diff options
| author | frosty <frostinator33@airmail.cc> | 2026-10-26 10:21:57 -0400 |
|---|---|---|
| committer | frosty <frostinator33@airmail.cc> | 2026-10-26 10:21:57 -0400 |
| commit | 74aecd8b0986d07b50b7ce310d64e645b8b54e38 (patch) | |
| tree | 560e6434f183eb54cd779936f3a4c9575146a066 /main/src/plugins | |
| parent | 3d7e90d401da02d430a87db0ec7e1db3a6f1a68f (diff) | |
| parent | 4436001bb361ad4f9e4e05819214ecf0ea77c063 (diff) | |
Merge 'upstream/master'
Diffstat (limited to 'main/src/plugins')
50 files changed, 7965 insertions, 0 deletions
diff --git a/main/src/plugins/omemo/dtls_srtp_verification_draft.vala b/main/src/plugins/omemo/dtls_srtp_verification_draft.vala new file mode 100644 index 00000000..09617a2a --- /dev/null +++ b/main/src/plugins/omemo/dtls_srtp_verification_draft.vala @@ -0,0 +1,268 @@ +using Omemo; +using Gee; +using Xmpp; + +namespace Dino.Plugins.Omemo.DtlsSrtpVerificationDraft { + public const string NS_URI = "http://gultsch.de/xmpp/drafts/omemo/dlts-srtp-verification"; + + public class StreamModule : XmppStreamModule { + + public static Xmpp.ModuleIdentity<StreamModule> IDENTITY = new Xmpp.ModuleIdentity<StreamModule>(NS_URI, "dtls_srtp_omemo_verification_draft"); + + private VerificationSendListener send_listener = new VerificationSendListener(); + private HashMap<string, int> device_id_by_jingle_sid = new HashMap<string, int>(); + private HashMap<string, int> device_id_by_muji_member = new HashMap<string, int>(); + private HashMap<string, Gee.List<string>> content_names_by_jingle_sid = new HashMap<string, Gee.List<string>>(); + + private void on_preprocess_incoming_iq_set_get(XmppStream stream, Xmpp.Iq.Stanza iq) { + if (iq.type_ != Iq.Stanza.TYPE_SET) return; + + Gee.List<StanzaNode> content_nodes = iq.stanza.get_deep_subnodes(Xep.Jingle.NS_URI + ":jingle", Xep.Jingle.NS_URI + ":content"); + if (content_nodes.size == 0) return; + + string? jingle_sid = iq.stanza.get_deep_attribute(Xep.Jingle.NS_URI + ":jingle", "sid"); + if (jingle_sid == null) return; + + Xep.Omemo.OmemoDecryptor decryptor = stream.get_module(Xep.Omemo.OmemoDecryptor.IDENTITY); + + foreach (StanzaNode content_node in content_nodes) { + string? content_name = content_node.get_attribute("name"); + if (content_name == null) continue; + StanzaNode? transport_node = content_node.get_subnode("transport", Xep.JingleIceUdp.NS_URI); + if (transport_node == null) continue; + StanzaNode? fingerprint_node = transport_node.get_subnode("fingerprint", NS_URI); + if (fingerprint_node == null) continue; + StanzaNode? encrypted_node = fingerprint_node.get_subnode("encrypted", Omemo.NS_URI); + if (encrypted_node == null) continue; + + Xep.Omemo.ParsedData? parsed_data = decryptor.parse_node(encrypted_node); + if (parsed_data == null || parsed_data.ciphertext == null) continue; + + if (device_id_by_jingle_sid.has_key(jingle_sid) && device_id_by_jingle_sid[jingle_sid] != parsed_data.sid) { + warning("Expected DTLS fingerprint to be OMEMO encrypted from %s %d, but it was from %d", iq.from.to_string(), device_id_by_jingle_sid[jingle_sid], parsed_data.sid); + } + + foreach (Bytes encr_key in parsed_data.our_potential_encrypted_keys.keys) { + parsed_data.is_prekey = parsed_data.our_potential_encrypted_keys[encr_key]; + parsed_data.encrypted_key = encr_key.get_data(); + + try { + uint8[] key = decryptor.decrypt_key(parsed_data, iq.from.bare_jid); + string cleartext = decryptor.decrypt(parsed_data.ciphertext, key, parsed_data.iv); + + StanzaNode new_fingerprint_node = new StanzaNode.build("fingerprint", Xep.JingleIceUdp.DTLS_NS_URI).add_self_xmlns() + .put_node(new StanzaNode.text(cleartext)); + string? hash_attr = fingerprint_node.get_attribute("hash", NS_URI); + string? setup_attr = fingerprint_node.get_attribute("setup", NS_URI); + if (hash_attr != null) new_fingerprint_node.put_attribute("hash", hash_attr); + if (setup_attr != null) new_fingerprint_node.put_attribute("setup", setup_attr); + transport_node.put_node(new_fingerprint_node); + + device_id_by_jingle_sid[jingle_sid] = parsed_data.sid; + if (!content_names_by_jingle_sid.has_key(content_name)) { + content_names_by_jingle_sid[content_name] = new ArrayList<string>(); + } + content_names_by_jingle_sid[content_name].add(content_name); + + stream.get_flag(Xep.Jingle.Flag.IDENTITY).get_session.begin(jingle_sid, (_, res) => { + Xep.Jingle.Session? session = stream.get_flag(Xep.Jingle.Flag.IDENTITY).get_session.end(res); + if (session == null || !session.contents_map.has_key(content_name)) return; + var encryption = new OmemoContentEncryption(NS_URI, "OMEMO", iq.from.bare_jid, device_id_by_jingle_sid[jingle_sid]); + session.contents_map[content_name].encryptions[NS_URI] = encryption; + + if (iq.stanza.get_deep_attribute(Xep.Jingle.NS_URI + ":jingle", "action") == "session-accept") { + session.additional_content_add_incoming.connect(on_content_add_received); + } + }); + + break; + } catch (Error e) { + debug("Decrypting message from %s/%d failed: %s", iq.from.bare_jid.to_string(), parsed_data.sid, e.message); + } + } + } + } + + private void on_preprocess_outgoing_iq_set_get(XmppStream stream, Xmpp.Iq.Stanza iq) { + if (iq.type_ != Iq.Stanza.TYPE_SET) return; + + StanzaNode? jingle_node = iq.stanza.get_subnode("jingle", Xep.Jingle.NS_URI); + if (jingle_node == null) return; + + int device_id = -1; + string? sid = jingle_node.get_attribute("sid", Xep.Jingle.NS_URI); + if (sid != null && device_id_by_jingle_sid.has_key(sid)) { + device_id = device_id_by_jingle_sid[sid]; + } + + StanzaNode? muji_node = jingle_node.get_subnode("muji", Xep.Muji.NS_URI); + if (muji_node != null) { + string muji_room = muji_node.get_attribute("room"); + try { + Jid muji_jid = Jid.from_string(muji_room); + if (device_id_by_muji_member.has_key(@"$(muji_jid.bare_jid)/$(iq.to)")) { + device_id = device_id_by_muji_member[@"$(muji_jid.bare_jid)/$(iq.to)"]; + } + } catch (InvalidJidError e) { + // Ignore + } + } + + if (device_id == -1) return; + + Gee.List<StanzaNode> content_nodes = jingle_node.get_subnodes("content", Xep.Jingle.NS_URI); + if (content_nodes.size == 0) return; + + foreach (StanzaNode content_node in content_nodes) { + StanzaNode? transport_node = content_node.get_subnode("transport", Xep.JingleIceUdp.NS_URI); + if (transport_node == null) continue; + StanzaNode? fingerprint_node = transport_node.get_subnode("fingerprint", Xep.JingleIceUdp.DTLS_NS_URI); + if (fingerprint_node == null) continue; + string fingerprint = fingerprint_node.get_deep_string_content(); + + StanzaNode? encrypted_node = null; + try { + Xep.Omemo.OmemoEncryptor encryptor = stream.get_module(Xep.Omemo.OmemoEncryptor.IDENTITY); + Xep.Omemo.EncryptionData enc_data = encryptor.encrypt_plaintext(fingerprint); + encryptor.encrypt_key(enc_data, iq.to.bare_jid, device_id); + encrypted_node = enc_data.get_encrypted_node(); + } catch (Error e) { + warning("Error while OMEMO-encrypting call keys: %s", e.message); + return; + } + + StanzaNode new_fingerprint_node = new StanzaNode.build("fingerprint", NS_URI).add_self_xmlns().put_node(encrypted_node); + string? hash_attr = fingerprint_node.get_attribute("hash", Xep.JingleIceUdp.DTLS_NS_URI); + string? setup_attr = fingerprint_node.get_attribute("setup", Xep.JingleIceUdp.DTLS_NS_URI); + if (hash_attr != null) new_fingerprint_node.put_attribute("hash", hash_attr); + if (setup_attr != null) new_fingerprint_node.put_attribute("setup", setup_attr); + transport_node.put_node(new_fingerprint_node); + + transport_node.sub_nodes.remove(fingerprint_node); + } + } + + private void on_message_received(XmppStream stream, Xmpp.MessageStanza message) { + StanzaNode? proceed_node = message.stanza.get_subnode("proceed", Xep.JingleMessageInitiation.NS_URI); + if (proceed_node == null) return; + + string? jingle_sid = proceed_node.get_attribute("id"); + if (jingle_sid == null) return; + + StanzaNode? device_node = proceed_node.get_subnode("device", NS_URI); + if (device_node == null) return; + + int device_id = device_node.get_attribute_int("id", -1); + if (device_id == -1) return; + + device_id_by_jingle_sid[jingle_sid] = device_id; + } + + private void on_session_initiate_received(XmppStream stream, Xep.Jingle.Session session) { + if (device_id_by_jingle_sid.has_key(session.sid)) { + foreach (Xep.Jingle.Content content in session.contents) { + on_content_add_received(stream, content); + } + } + session.additional_content_add_incoming.connect(on_content_add_received); + } + + private void on_content_add_received(XmppStream stream, Xep.Jingle.Content content) { + if (!content_names_by_jingle_sid.has_key(content.session.sid) || content_names_by_jingle_sid[content.session.sid].contains(content.content_name)) { + var encryption = new OmemoContentEncryption(NS_URI, "OMEMO", content.peer_full_jid.bare_jid, device_id_by_jingle_sid[content.session.sid]); + content.encryptions[encryption.encryption_ns] = encryption; + } + } + + private void on_pre_send_presence_stanza(XmppStream stream, Presence.Stanza presence) { + StanzaNode? muji_node = presence.stanza.get_subnode("muji", Xep.Muji.NS_URI); + if (muji_node == null) return; + + StanzaNode device_node = new StanzaNode.build("device", NS_URI).add_self_xmlns() + .put_attribute("id", stream.get_module(Omemo.StreamModule.IDENTITY).store.local_registration_id.to_string()); + muji_node.put_node(device_node); + } + + private void on_received_available(XmppStream stream, Presence.Stanza presence) { + StanzaNode? muji_node = presence.stanza.get_subnode("muji", Xep.Muji.NS_URI); + if (muji_node == null) return; + + StanzaNode? device_node = muji_node.get_subnode("device", NS_URI); + if (device_node == null) return; + + int device_id = device_node.get_attribute_int("id", -1); + if (device_id == -1) return; + + StanzaNode? muc_x_node = presence.stanza.get_subnode("x", "http://jabber.org/protocol/muc#user"); + if (muc_x_node == null) return; + + StanzaNode? item_node = muc_x_node.get_subnode("item"); + if (item_node == null) return; + + Jid? real_jid = null; + try { + string jid_attribute = item_node.get_attribute("jid"); + if (jid_attribute == null) return; + real_jid = Jid.from_string(jid_attribute); + } catch (InvalidJidError e) { + // Ignore + return; + } + + device_id_by_muji_member[@"$(presence.from.bare_jid)/$(real_jid)"] = device_id; + } + + public override void attach(XmppStream stream) { + stream.get_module(Xmpp.MessageModule.IDENTITY).received_message.connect(on_message_received); + stream.get_module(Xmpp.MessageModule.IDENTITY).send_pipeline.connect(send_listener); + stream.get_module(Xmpp.Iq.Module.IDENTITY).preprocess_incoming_iq_set_get.connect(on_preprocess_incoming_iq_set_get); + stream.get_module(Xmpp.Iq.Module.IDENTITY).preprocess_outgoing_iq_set_get.connect(on_preprocess_outgoing_iq_set_get); + stream.get_module(Xep.Jingle.Module.IDENTITY).session_initiate_received.connect(on_session_initiate_received); + stream.get_module(Xmpp.Presence.Module.IDENTITY).pre_send_presence_stanza.connect(on_pre_send_presence_stanza); + stream.get_module(Xmpp.Presence.Module.IDENTITY).received_available.connect(on_received_available); + } + + public override void detach(XmppStream stream) { + stream.get_module(Xmpp.MessageModule.IDENTITY).received_message.disconnect(on_message_received); + stream.get_module(Xmpp.MessageModule.IDENTITY).send_pipeline.disconnect(send_listener); + stream.get_module(Xmpp.Iq.Module.IDENTITY).preprocess_incoming_iq_set_get.disconnect(on_preprocess_incoming_iq_set_get); + stream.get_module(Xmpp.Iq.Module.IDENTITY).preprocess_outgoing_iq_set_get.disconnect(on_preprocess_outgoing_iq_set_get); + stream.get_module(Xep.Jingle.Module.IDENTITY).session_initiate_received.disconnect(on_session_initiate_received); + stream.get_module(Xmpp.Presence.Module.IDENTITY).received_available.disconnect(on_received_available); + } + + public override string get_ns() { return NS_URI; } + + public override string get_id() { return IDENTITY.id; } + } + + public class VerificationSendListener : StanzaListener<MessageStanza> { + + private string[] after_actions_const = {}; + + public override string action_group { get { return "REWRITE_NODES"; } } + public override string[] after_actions { get { return after_actions_const; } } + + public override async bool run(XmppStream stream, MessageStanza message) { + StanzaNode? proceed_node = message.stanza.get_subnode("proceed", Xep.JingleMessageInitiation.NS_URI); + if (proceed_node == null) return false; + + StanzaNode device_node = new StanzaNode.build("device", NS_URI).add_self_xmlns() + .put_attribute("id", stream.get_module(Omemo.StreamModule.IDENTITY).store.local_registration_id.to_string()); + proceed_node.put_node(device_node); + return false; + } + } + + public class OmemoContentEncryption : Xep.Jingle.ContentEncryption { + public Jid jid { get; set; } + public int sid { get; set; } + + public OmemoContentEncryption(string encryption_ns, string encryption_name, Jid jid, int sid) { + base(encryption_ns, encryption_name); + this.jid = jid; + this.sid = sid; + } + } +} + diff --git a/main/src/plugins/omemo/file_transfer/file_decryptor.vala b/main/src/plugins/omemo/file_transfer/file_decryptor.vala new file mode 100644 index 00000000..8ae0b96b --- /dev/null +++ b/main/src/plugins/omemo/file_transfer/file_decryptor.vala @@ -0,0 +1,90 @@ +using Dino.Entities; + +using Dino.Crypto; +using Omemo; + +namespace Dino.Plugins.Omemo { + +public class OmemoHttpFileReceiveData : HttpFileReceiveData { + public string original_url; +} + +public class OmemoFileDecryptor : FileDecryptor, Object { + + private Regex url_regex = /^aesgcm:\/\/(.*)#(([A-Fa-f0-9]{2}){48}|([A-Fa-f0-9]{2}){44})$/; + + public Encryption get_encryption() { + return Encryption.OMEMO; + } + + public FileReceiveData prepare_get_meta_info(Conversation conversation, FileTransfer file_transfer, FileReceiveData receive_data) { + HttpFileReceiveData? http_receive_data = receive_data as HttpFileReceiveData; + if (http_receive_data == null) assert(false); + if ((receive_data as OmemoHttpFileReceiveData) != null) return receive_data; + + var omemo_http_receive_data = new OmemoHttpFileReceiveData(); + omemo_http_receive_data.url = aesgcm_to_https_link(http_receive_data.url); + omemo_http_receive_data.original_url = http_receive_data.url; + + return omemo_http_receive_data; + } + + public FileMeta prepare_download_file(Conversation conversation, FileTransfer file_transfer, FileReceiveData receive_data, FileMeta file_meta) { + if (file_meta.file_name != null) { + file_meta.file_name = file_meta.file_name.split("#")[0]; + } + return file_meta; + } + + public bool can_decrypt_file(Conversation conversation, FileTransfer file_transfer, FileReceiveData receive_data) { + HttpFileReceiveData? http_file_receive = receive_data as HttpFileReceiveData; + if (http_file_receive == null) return false; + + return this.url_regex.match(http_file_receive.url) || (receive_data as OmemoHttpFileReceiveData) != null; + } + + public async InputStream decrypt_file(InputStream encrypted_stream, Conversation conversation, FileTransfer file_transfer, FileReceiveData receive_data) throws FileReceiveError { + const uint KEY_SIZE = 32; + try { + OmemoHttpFileReceiveData? omemo_http_receive_data = receive_data as OmemoHttpFileReceiveData; + if (omemo_http_receive_data == null) assert(false); + + // Decode IV and key + MatchInfo match_info; + this.url_regex.match(omemo_http_receive_data.original_url, 0, out match_info); + uint8[] iv_and_key = hex_to_bin(match_info.fetch(2).up()); + uint8[] iv = iv_and_key[0:iv_and_key.length-KEY_SIZE]; + uint8[] key = iv_and_key[iv_and_key.length-KEY_SIZE:iv_and_key.length]; + + file_transfer.encryption = Encryption.OMEMO; + debug("Decrypting file %s from %s", file_transfer.file_name, file_transfer.server_file_name); + + SymmetricCipherDecryption cipher = create_symmetric_cipher_decryption("AES-GCM"); + cipher.set_key(key); + cipher.set_iv(iv); + return new ConverterInputStream(encrypted_stream, new SymmetricCipherDecrypter((owned) cipher, 16)); + + } catch (Crypto.Error e) { + throw new FileReceiveError.DECRYPTION_FAILED("OMEMO file decryption error: %s".printf(e.message)); + } catch (GLib.Error e) { + throw new FileReceiveError.DECRYPTION_FAILED("OMEMO file decryption error: %s".printf(e.message)); + } + } + + private uint8[] hex_to_bin(string hex) { + uint8[] bin = new uint8[hex.length / 2]; + const string HEX = "0123456789ABCDEF"; + for (int i = 0; i < hex.length / 2; i++) { + bin[i] = (uint8) (HEX.index_of_char(hex[i*2]) << 4) | HEX.index_of_char(hex[i*2+1]); + } + return bin; + } + + private string aesgcm_to_https_link(string aesgcm_link) { + MatchInfo match_info; + this.url_regex.match(aesgcm_link, 0, out match_info); + return "https://" + match_info.fetch(1); + } +} + +} diff --git a/main/src/plugins/omemo/file_transfer/file_encryptor.vala b/main/src/plugins/omemo/file_transfer/file_encryptor.vala new file mode 100644 index 00000000..5f1d6e69 --- /dev/null +++ b/main/src/plugins/omemo/file_transfer/file_encryptor.vala @@ -0,0 +1,76 @@ +using Gee; +using Gtk; + +using Dino.Crypto; +using Dino.Entities; +using Xmpp; +using Omemo; + +namespace Dino.Plugins.Omemo { + +public class OmemoHttpFileMeta : HttpFileMeta { + public uint8[] iv; + public uint8[] key; +} + +public class OmemoFileEncryptor : Dino.FileEncryptor, Object { + + public bool can_encrypt_file(Conversation conversation, FileTransfer file_transfer) { + return file_transfer.encryption == Encryption.OMEMO; + } + + public FileMeta encrypt_file(Conversation conversation, FileTransfer file_transfer) throws FileSendError { + const uint KEY_SIZE = 32; + const uint IV_SIZE = 12; + var omemo_http_file_meta = new OmemoHttpFileMeta(); + + try { + //Create a key and use it to encrypt the file + uint8[] iv = new uint8[IV_SIZE]; + Crypto.randomize(iv); + uint8[] key = new uint8[KEY_SIZE]; + Crypto.randomize(key); + + SymmetricCipherEncryption cipher = create_symmetric_cipher_encryption("AES-GCM"); + cipher.set_key(key); + cipher.set_iv(iv); + + omemo_http_file_meta.iv = iv; + omemo_http_file_meta.key = key; + omemo_http_file_meta.size = file_transfer.size + 16; + omemo_http_file_meta.content_type = new FileContentType.from_mime_type("application/octet-stream"); + file_transfer.input_stream = new ConverterInputStream(file_transfer.input_stream, new SymmetricCipherEncrypter((owned) cipher, 16)); + } catch (Crypto.Error error) { + throw new FileSendError.ENCRYPTION_FAILED("OMEMO file encryption error: %s".printf(error.message)); + } catch (GLib.Error error) { + throw new FileSendError.ENCRYPTION_FAILED("OMEMO file encryption error: %s".printf(error.message)); + } + + debug("Encrypting file %s as %s", file_transfer.file_name, file_transfer.server_file_name); + + return omemo_http_file_meta; + } + + public FileSendData? preprocess_send_file(Conversation conversation, FileTransfer file_transfer, FileSendData file_send_data, FileMeta file_meta) { + HttpFileSendData? send_data = file_send_data as HttpFileSendData; + if (send_data == null) return null; + + OmemoHttpFileMeta? omemo_http_file_meta = file_meta as OmemoHttpFileMeta; + if (omemo_http_file_meta == null) return null; + + // Convert iv and key to hex + string iv_and_key = ""; + foreach (uint8 byte in omemo_http_file_meta.iv) iv_and_key += byte.to_string("%02x"); + foreach (uint8 byte in omemo_http_file_meta.key) iv_and_key += byte.to_string("%02x"); + + string aesgcm_link = send_data.url_down + "#" + iv_and_key; + aesgcm_link = "aesgcm://" + aesgcm_link.substring(8); // replace https:// by aesgcm:// + + send_data.url_down = aesgcm_link; + send_data.encrypt_message = true; + + return file_send_data; + } +} + +} diff --git a/main/src/plugins/omemo/jingle/jet_omemo.vala b/main/src/plugins/omemo/jingle/jet_omemo.vala new file mode 100644 index 00000000..07d03f64 --- /dev/null +++ b/main/src/plugins/omemo/jingle/jet_omemo.vala @@ -0,0 +1,115 @@ +using Dino.Crypto; +using Dino.Crypto.Omemo; +using Dino; +using Dino.Entities; +using Gee; +using Omemo; +using Xmpp; +using Xmpp.Xep; + +namespace Dino.Plugins.JetOmemo { + +private const string NS_URI = "urn:xmpp:jingle:jet-omemo:0"; +private const string AES_128_GCM_URI = "urn:xmpp:ciphers:aes-128-gcm-nopadding"; + +public class Module : XmppStreamModule, Jet.EnvelopEncoding { + public static Xmpp.ModuleIdentity<Module> IDENTITY = new Xmpp.ModuleIdentity<Module>(NS_URI, "0396_jet_omemo"); + const uint KEY_SIZE = 16; + const uint IV_SIZE = 12; + + public override void attach(XmppStream stream) { + if (stream.get_module(Jet.Module.IDENTITY) != null) { + stream.get_module(ServiceDiscovery.Module.IDENTITY).add_feature(stream, NS_URI); + stream.get_module(Jet.Module.IDENTITY).register_envelop_encoding(this); + stream.get_module(Jet.Module.IDENTITY).register_cipher(new AesGcmCipher(KEY_SIZE, IV_SIZE, AES_128_GCM_URI)); + } + } + + public override void detach(XmppStream stream) { + stream.get_module(ServiceDiscovery.Module.IDENTITY).remove_feature(stream, NS_URI); + } + + public async bool is_available(XmppStream stream, Jid full_jid) { + bool? has_feature = yield stream.get_module(ServiceDiscovery.Module.IDENTITY).has_entity_feature(stream, full_jid, NS_URI); + if (has_feature == null || !(!)has_feature) { + return false; + } + return yield stream.get_module(Xep.Jet.Module.IDENTITY).is_available(stream, full_jid); + } + + public string get_type_uri() { + return Omemo.NS_URI; + } + + public Jet.TransportSecret decode_envolop(XmppStream stream, Jid local_full_jid, Jid peer_full_jid, StanzaNode security) throws Jingle.IqError { + StanzaNode? encrypted = security.get_subnode("encrypted", Omemo.NS_URI); + if (encrypted == null) throw new Jingle.IqError.BAD_REQUEST("Invalid JET-OMEMO envelop: missing encrypted element"); + + Xep.Omemo.OmemoDecryptor decryptor = stream.get_module(Xep.Omemo.OmemoDecryptor.IDENTITY); + + Xmpp.Xep.Omemo.ParsedData? data = decryptor.parse_node(encrypted); + if (data == null) throw new Jingle.IqError.BAD_REQUEST("Invalid JET-OMEMO envelop: bad encrypted element"); + + foreach (Bytes encr_key in data.our_potential_encrypted_keys.keys) { + data.is_prekey = data.our_potential_encrypted_keys[encr_key]; + data.encrypted_key = encr_key.get_data(); + + try { + uint8[] key = decryptor.decrypt_key(data, peer_full_jid.bare_jid); + return new Jet.TransportSecret(key, data.iv); + } catch (GLib.Error e) { + debug("Decrypting JET key from %s/%d failed: %s", peer_full_jid.bare_jid.to_string(), data.sid, e.message); + } + } + throw new Jingle.IqError.NOT_ACCEPTABLE("Not encrypted for targeted device"); + } + + public void encode_envelop(XmppStream stream, Jid local_full_jid, Jid peer_full_jid, Jet.SecurityParameters security_params, StanzaNode security) { + Store store = stream.get_module(Omemo.StreamModule.IDENTITY).store; + + var encryption_data = new Xep.Omemo.EncryptionData(store.local_registration_id); + encryption_data.iv = security_params.secret.initialization_vector; + encryption_data.keytag = security_params.secret.transport_key; + Xep.Omemo.OmemoEncryptor encryptor = stream.get_module(Xep.Omemo.OmemoEncryptor.IDENTITY); + encryptor.encrypt_key_to_recipient(stream, encryption_data, peer_full_jid.bare_jid); + + security.put_node(encryption_data.get_encrypted_node()); + } + + public override string get_ns() { return NS_URI; } + public override string get_id() { return IDENTITY.id; } +} + +public class AesGcmCipher : Jet.Cipher, Object { + private uint key_size; + private uint default_iv_size; + private string uri; + public AesGcmCipher(uint key_size, uint default_iv_size, string uri) { + this.key_size = key_size; + this.default_iv_size = default_iv_size; + this.uri = uri; + } + public string get_cipher_uri() { + return uri; + } + public Jet.TransportSecret generate_random_secret() { + uint8[] iv = new uint8[default_iv_size]; + Crypto.randomize(iv); + uint8[] key = new uint8[key_size]; + Crypto.randomize(key); + return new Jet.TransportSecret(key, iv); + } + public InputStream wrap_input_stream(InputStream input, Jet.TransportSecret secret) requires (secret.transport_key.length == key_size) { + SymmetricCipherDecryption cipher = create_symmetric_cipher_decryption("AES-GCM"); + cipher.set_key(secret.transport_key); + cipher.set_iv(secret.initialization_vector); + return new ConverterInputStream(input, new SymmetricCipherDecrypter((owned) cipher, 16)); + } + public OutputStream wrap_output_stream(OutputStream output, Jet.TransportSecret secret) requires (secret.transport_key.length == key_size) { + SymmetricCipherEncryption cipher = create_symmetric_cipher_encryption("AES-GCM"); + cipher.set_key(secret.transport_key); + cipher.set_iv(secret.initialization_vector); + return new ConverterOutputStream(output, new SymmetricCipherEncrypter((owned) cipher, 16)); + } +} +} diff --git a/main/src/plugins/omemo/jingle/jingle_helper.vala b/main/src/plugins/omemo/jingle/jingle_helper.vala new file mode 100644 index 00000000..ec6c0eca --- /dev/null +++ b/main/src/plugins/omemo/jingle/jingle_helper.vala @@ -0,0 +1,53 @@ +using Dino.Entities; +using Xmpp; + +namespace Dino.Plugins.JetOmemo { +public class EncryptionHelper : JingleFileEncryptionHelper, Object { + private StreamInteractor stream_interactor; + + public EncryptionHelper(StreamInteractor stream_interactor) { + this.stream_interactor = stream_interactor; + } + + public bool can_transfer(Conversation conversation) { + return true; + } + + public async bool can_encrypt(Conversation conversation, FileTransfer file_transfer, Jid? full_jid) { + XmppStream? stream = stream_interactor.get_stream(conversation.account); + if (stream == null) return false; + + Gee.List<Jid>? resources = stream.get_flag(Presence.Flag.IDENTITY).get_resources(conversation.counterpart); + if (resources == null) return false; + + if (full_jid == null) { + foreach (Jid test_jid in resources) { + if (yield stream.get_module(Module.IDENTITY).is_available(stream, test_jid)) { + return true; + } + } + } else { + if (yield stream.get_module(Module.IDENTITY).is_available(stream, full_jid)) { + return true; + } + } + return false; + } + + public string? get_precondition_name(Conversation conversation, FileTransfer file_transfer) { + return Xep.Jet.NS_URI; + } + + public Object? get_precondition_options(Conversation conversation, FileTransfer file_transfer) { + return new Xep.Jet.Options(Omemo.NS_URI, AES_128_GCM_URI); + } + + public Encryption get_encryption(Xmpp.Xep.JingleFileTransfer.FileTransfer jingle_transfer) { + Xep.Jet.SecurityParameters? security = jingle_transfer.security as Xep.Jet.SecurityParameters; + if (security != null && security.encoding.get_type_uri() == Omemo.NS_URI) { + return Encryption.OMEMO; + } + return Encryption.NONE; + } +} +} diff --git a/main/src/plugins/omemo/logic/database.vala b/main/src/plugins/omemo/logic/database.vala new file mode 100644 index 00000000..f2a92259 --- /dev/null +++ b/main/src/plugins/omemo/logic/database.vala @@ -0,0 +1,284 @@ +using Gee; + +using Dino.Entities; +using Dino.Qlite; + +namespace Dino.Plugins.Omemo { + +public class Database : Qlite.Database { + private const int VERSION = 5; + + public class IdentityMetaTable : Table { + //Default to provide backwards compatability + public Column<int> identity_id = new Column.Integer("identity_id") { not_null = true, min_version = 2, default = "-1" }; + public Column<string> address_name = new Column.Text("address_name") { not_null = true }; + public Column<int> device_id = new Column.Integer("device_id") { not_null = true }; + public Column<string?> identity_key_public_base64 = new Column.Text("identity_key_public_base64"); + public Column<bool> trusted_identity = new Column.BoolInt("trusted_identity") { default = "0", max_version = 1 }; + public Column<int> trust_level = new Column.Integer("trust_level") { default = TrustLevel.UNKNOWN.to_string(), min_version = 2 }; + public Column<bool> now_active = new Column.BoolInt("now_active") { default = "1" }; + public Column<long> last_active = new Column.Long("last_active"); + public Column<long> last_message_untrusted = new Column.Long("last_message_untrusted") { min_version = 5 }; + public Column<long> last_message_undecryptable = new Column.Long("last_message_undecryptable") { min_version = 5 }; + + internal IdentityMetaTable(Database db) { + base(db, "identity_meta"); + init({identity_id, address_name, device_id, identity_key_public_base64, trusted_identity, trust_level, now_active, last_active, last_message_untrusted, last_message_undecryptable}); + index("identity_meta_idx", {identity_id, address_name, device_id}, true); + index("identity_meta_list_idx", {identity_id, address_name}); + } + + public QueryBuilder with_address(int identity_id, string address_name) { + return select().with(this.identity_id, "=", identity_id).with(this.address_name, "=", address_name); + } + + public QueryBuilder get_with_device_id(int identity_id, int device_id) { + return select().with(this.identity_id, "=", identity_id).with(this.device_id, "=", device_id); + } + + public void insert_device_list(int32 identity_id, string address_name, ArrayList<int32> devices) { + update().with(this.identity_id, "=", identity_id).with(this.address_name, "=", address_name).set(now_active, false).perform(); + foreach (int32 device_id in devices) { + upsert() + .value(this.identity_id, identity_id, true) + .value(this.address_name, address_name, true) + .value(this.device_id, device_id, true) + .value(this.now_active, true) + .value(this.last_active, (long) new DateTime.now_utc().to_unix()) + .perform(); + } + } + + public int64 insert_device_bundle(int32 identity_id, string address_name, int device_id, Bundle bundle, TrustLevel trust) { + if (bundle == null || bundle.identity_key == null) return -1; + // Do not replace identity_key if it was known before, it should never change! + string identity_key = Base64.encode(bundle.identity_key.serialize()); + RowOption row = with_address(identity_id, address_name).with(this.device_id, "=", device_id).single().row(); + if (row.is_present() && row[identity_key_public_base64] != null && row[identity_key_public_base64] != identity_key) { + critical("Tried to change the identity key for a known device id. Likely an attack."); + return -1; + } + return upsert() + .value(this.identity_id, identity_id, true) + .value(this.address_name, address_name, true) + .value(this.device_id, device_id, true) + .value(this.identity_key_public_base64, identity_key) + .value(this.trust_level, trust).perform(); + } + + public int64 insert_device_session(int32 identity_id, string address_name, int device_id, string identity_key, TrustLevel trust) { + RowOption row = with_address(identity_id, address_name).with(this.device_id, "=", device_id).single().row(); + if (row.is_present() && row[identity_key_public_base64] != null && row[identity_key_public_base64] != identity_key) { + critical("Tried to change the identity key for a known device id. Likely an attack."); + return -1; + } + return upsert() + .value(this.identity_id, identity_id, true) + .value(this.address_name, address_name, true) + .value(this.device_id, device_id, true) + .value(this.identity_key_public_base64, identity_key) + .value(this.trust_level, trust).perform(); + } + + public void update_last_message_untrusted(int identity_id, int device_id, DateTime? time) { + var stmt = update() + .with(this.identity_id, "=", identity_id) + .with(this.device_id, "=", device_id); + if (time != null) { + stmt.set(last_message_untrusted, (long)time.to_unix()); + } else { + stmt.set_null(last_message_untrusted); + } + stmt.perform(); + } + + public void update_last_message_undecryptable(int identity_id, int device_id, DateTime? time) { + var stmt = update() + .with(this.identity_id, "=", identity_id) + .with(this.device_id, "=", device_id); + if (time != null) { + stmt.set(last_message_undecryptable, (long)time.to_unix()); + } else { + stmt.set_null(last_message_undecryptable); + } + stmt.perform(); + } + + public QueryBuilder get_trusted_devices(int identity_id, string address_name) { + return this.with_address(identity_id, address_name) + .with(this.trust_level, "!=", TrustLevel.UNTRUSTED) + .with(this.now_active, "=", true); + } + + public QueryBuilder get_known_devices(int identity_id, string address_name) { + return this.with_address(identity_id, address_name) + .with(this.trust_level, "!=", TrustLevel.UNKNOWN) + .without_null(this.identity_key_public_base64); + } + + public QueryBuilder get_unknown_devices(int identity_id, string address_name) { + return this.with_address(identity_id, address_name) + .with_null(this.identity_key_public_base64); + } + + public QueryBuilder get_new_devices(int identity_id, string address_name) { + return this.with_address(identity_id, address_name) + .with(this.trust_level, "=", TrustLevel.UNKNOWN) + .without_null(this.identity_key_public_base64); + } + + public Row? get_device(int identity_id, string address_name, int device_id) { + return this.with_address(identity_id, address_name) + .with(this.device_id, "=", device_id).single().row().inner; + } + } + + + public class TrustTable : Table { + public Column<int> identity_id = new Column.Integer("identity_id") { not_null = true }; + public Column<string> address_name = new Column.Text("address_name"); + public Column<bool> blind_trust = new Column.BoolInt("blind_trust") { default = "1" } ; + + internal TrustTable(Database db) { + base(db, "trust"); + init({identity_id, address_name, blind_trust}); + index("trust_idx", {identity_id, address_name}, true); + } + + public bool get_blind_trust(int32 identity_id, string address_name, bool def = false) { + RowOption row = this.select().with(this.identity_id, "=", identity_id) + .with(this.address_name, "=", address_name).single().row(); + if (row.is_present()) return row[blind_trust]; + return def; + } + } + + public class IdentityTable : Table { + public Column<int> id = new Column.Integer("id") { primary_key = true, auto_increment = true }; + public Column<int> account_id = new Column.Integer("account_id") { unique = true, not_null = true }; + public Column<int> device_id = new Column.Integer("device_id") { not_null = true }; + public Column<string> identity_key_private_base64 = new Column.NonNullText("identity_key_private_base64"); + public Column<string> identity_key_public_base64 = new Column.NonNullText("identity_key_public_base64"); + + internal IdentityTable(Database db) { + base(db, "identity"); + init({id, account_id, device_id, identity_key_private_base64, identity_key_public_base64}); + } + + public int get_id(int account_id) { + int id = -1; + Row? row = this.row_with(this.account_id, account_id).inner; + if (row != null) id = ((!)row)[this.id]; + return id; + } + } + + public class SignedPreKeyTable : Table { + public Column<int> identity_id = new Column.Integer("identity_id") { not_null = true }; + public Column<int> signed_pre_key_id = new Column.Integer("signed_pre_key_id") { not_null = true }; + public Column<string> record_base64 = new Column.NonNullText("record_base64"); + + internal SignedPreKeyTable(Database db) { + base(db, "signed_pre_key"); + init({identity_id, signed_pre_key_id, record_base64}); + unique({identity_id, signed_pre_key_id}); + index("signed_pre_key_idx", {identity_id, signed_pre_key_id}, true); + } + } + + public class PreKeyTable : Table { + public Column<int> identity_id = new Column.Integer("identity_id") { not_null = true }; + public Column<int> pre_key_id = new Column.Integer("pre_key_id") { not_null = true }; + public Column<string> record_base64 = new Column.NonNullText("record_base64"); + + internal PreKeyTable(Database db) { + base(db, "pre_key"); + init({identity_id, pre_key_id, record_base64}); + unique({identity_id, pre_key_id}); + index("pre_key_idx", {identity_id, pre_key_id}, true); + } + } + + public class SessionTable : Table { + public Column<int> identity_id = new Column.Integer("identity_id") { not_null = true }; + public Column<string> address_name = new Column.NonNullText("name"); + public Column<int> device_id = new Column.Integer("device_id") { not_null = true }; + public Column<string> record_base64 = new Column.NonNullText("record_base64"); + + internal SessionTable(Database db) { + base(db, "session"); + init({identity_id, address_name, device_id, record_base64}); + unique({identity_id, address_name, device_id}); + index("session_idx", {identity_id, address_name, device_id}, true); + } + } + + public class ContentItemMetaTable : Table { + public Column<int> content_item_id = new Column.Integer("message_id") { primary_key = true }; + public Column<int> identity_id = new Column.Integer("identity_id") { not_null = true }; + public Column<string> address_name = new Column.Text("address_name") { not_null = true }; + public Column<int> device_id = new Column.Integer("device_id") { not_null = true }; + public Column<bool> trusted_when_received = new Column.BoolInt("trusted_when_received") { not_null = true, default = "1" }; + + internal ContentItemMetaTable(Database db) { + base(db, "content_item_meta"); + init({content_item_id, identity_id, address_name, device_id, trusted_when_received}); + index("content_item_meta_device_idx", {identity_id, device_id, address_name}); + } + + public RowOption with_content_item(ContentItem item) { + return row_with(content_item_id, item.id); + } + + public QueryBuilder with_device(int identity_id, string address_name, int device_id) { + return select() + .with(this.identity_id, "=", identity_id) + .with(this.address_name, "=", address_name) + .with(this.device_id, "=", device_id); + } + } + + public IdentityMetaTable identity_meta { get; private set; } + public TrustTable trust { get; private set; } + public IdentityTable identity { get; private set; } + public SignedPreKeyTable signed_pre_key { get; private set; } + public PreKeyTable pre_key { get; private set; } + public SessionTable session { get; private set; } + public ContentItemMetaTable content_item_meta { get; private set; } + + public Database(string fileName) { + base(fileName, VERSION); + identity_meta = new IdentityMetaTable(this); + trust = new TrustTable(this); + identity = new IdentityTable(this); + signed_pre_key = new SignedPreKeyTable(this); + pre_key = new PreKeyTable(this); + session = new SessionTable(this); + content_item_meta = new ContentItemMetaTable(this); + init({identity_meta, trust, identity, signed_pre_key, pre_key, session, content_item_meta}); + + try { + exec("PRAGMA journal_mode = WAL"); + exec("PRAGMA synchronous = NORMAL"); + exec("PRAGMA secure_delete = ON"); + } catch (Error e) { + error("Failed to set OMEMO database properties: %s", e.message); + } + } + + public override void migrate(long oldVersion) { + if(oldVersion == 1) { + try { + exec("DROP INDEX identity_meta_idx"); + exec("DROP INDEX identity_meta_list_idx"); + exec("CREATE UNIQUE INDEX identity_meta_idx ON identity_meta (identity_id, address_name, device_id)"); + exec("CREATE INDEX identity_meta_list_idx ON identity_meta (identity_id, address_name)"); + } catch (Error e) { + stderr.printf("Failed to migrate OMEMO database\n"); + Process.exit(-1); + } + } + } +} + +} diff --git a/main/src/plugins/omemo/logic/decrypt.vala b/main/src/plugins/omemo/logic/decrypt.vala new file mode 100644 index 00000000..135ac42c --- /dev/null +++ b/main/src/plugins/omemo/logic/decrypt.vala @@ -0,0 +1,212 @@ +using Gee; + +using Dino.Crypto.Omemo; +using Dino.Entities; +using Dino.Qlite; +using Omemo; +using Xmpp; + +namespace Dino.Plugins.Omemo { + + public class OmemoDecryptor : Xep.Omemo.OmemoDecryptor { + + private Account account; + private Store store; + private Database db; + private StreamInteractor stream_interactor; + private TrustManager trust_manager; + + public override uint32 own_device_id { get { return store.local_registration_id; }} + + public OmemoDecryptor(Account account, StreamInteractor stream_interactor, TrustManager trust_manager, Database db, Store store) { + this.account = account; + this.stream_interactor = stream_interactor; + this.trust_manager = trust_manager; + this.db = db; + this.store = store; + } + + public bool decrypt_message(Entities.Message message, Xmpp.MessageStanza stanza, Conversation conversation) { + StanzaNode? encrypted_node = stanza.stanza.get_subnode("encrypted", NS_URI); + if (encrypted_node == null || MessageFlag.get_flag(stanza) != null || stanza.from == null) return false; + + if (!Plugin.ensure_context()) return false; + int identity_id = db.identity.get_id(conversation.account.id); + + MessageFlag flag = new MessageFlag(); + stanza.add_flag(flag); + + Xep.Omemo.ParsedData? data = parse_node(encrypted_node); + if (data == null) return false; + + + foreach (Bytes encr_key in data.our_potential_encrypted_keys.keys) { + data.is_prekey = data.our_potential_encrypted_keys[encr_key]; + data.encrypted_key = encr_key.get_data(); + Gee.List<Jid> possible_jids = get_potential_message_jids(message, data, identity_id); + if (possible_jids.size == 0) { + debug("Received message from unknown entity with device id %d", data.sid); + } + + foreach (Jid possible_jid in possible_jids) { + try { + uint8[] key = decrypt_key(data, possible_jid); + if (data.ciphertext != null) { + string cleartext = arr_to_str(aes_decrypt(Cipher.AES_GCM_NOPADDING, key, data.iv, data.ciphertext)); + message.body = cleartext; + } + + // If we figured out which real jid a message comes from due to decryption working, save it + if (conversation.type_ == Conversation.Type.GROUPCHAT && message.real_jid == null) { + message.real_jid = possible_jid; + } + + message.encryption = Encryption.OMEMO; + + trust_manager.message_device_id_map[message] = data.sid; + return true; + } catch (Error e) { + debug("Decrypting message from %s/%d failed: %s", possible_jid.to_string(), data.sid, e.message); + } + } + } + + if ( + data.ciphertext != null && // Ratchet forwarding doesn't contain payload and might not include us, which is ok + data.our_potential_encrypted_keys.size == 0 && // The message was not encrypted to us + stream_interactor.module_manager.get_module(message.account, StreamModule.IDENTITY).store.local_registration_id != data.sid // Message from this device. Never encrypted to itself. + ) { + db.identity_meta.update_last_message_undecryptable(identity_id, data.sid, message.time); + trust_manager.bad_message_state_updated(conversation.account, message.from, data.sid); + } + + debug("Received OMEMO encryped message that could not be decrypted."); + return false; + } + + public Gee.List<Jid> get_potential_message_jids(Entities.Message message, Xmpp.Xep.Omemo.ParsedData data, int identity_id) { + Gee.List<Jid> possible_jids = new ArrayList<Jid>(); + if (message.type_ == Message.Type.CHAT) { + possible_jids.add(message.from.bare_jid); + } else { + if (message.real_jid != null) { + possible_jids.add(message.real_jid.bare_jid); + } else if (data.is_prekey) { + // pre key messages do store the identity key, so we can use that to find the real jid + PreKeyOmemoMessage msg = Plugin.get_context().deserialize_signal_pre_key_message(data.encrypted_key); + string identity_key = Base64.encode(msg.identity_key.serialize()); + foreach (Row row in db.identity_meta.get_with_device_id(identity_id, data.sid).with(db.identity_meta.identity_key_public_base64, "=", identity_key)) { + try { + possible_jids.add(Jid.from_string(row[db.identity_meta.address_name])); + } catch (InvalidJidError e) { + warning("Ignoring invalid jid from database: %s", e.message); + } + } + } else { + // If we don't know the device name (MUC history w/o MAM), test decryption with all keys with fitting device id + foreach (Row row in db.identity_meta.get_with_device_id(identity_id, data.sid)) { + try { + possible_jids.add(Jid.from_string(row[db.identity_meta.address_name])); + } catch (InvalidJidError e) { + warning("Ignoring invalid jid from database: %s", e.message); + } + } + } + } + return possible_jids; + } + + public override uint8[] decrypt_key(Xmpp.Xep.Omemo.ParsedData data, Jid from_jid) throws GLib.Error { + int sid = data.sid; + uint8[] ciphertext = data.ciphertext; + uint8[] encrypted_key = data.encrypted_key; + + Address address = new Address(from_jid.to_string(), sid); + uint8[] key; + + if (data.is_prekey) { + int identity_id = db.identity.get_id(account.id); + PreKeyOmemoMessage msg = Plugin.get_context().deserialize_signal_pre_key_message(encrypted_key); + string identity_key = Base64.encode(msg.identity_key.serialize()); + + bool ok = update_db_for_prekey(identity_id, identity_key, from_jid, sid); + if (!ok) throw new GLib.Error(-1, 0, "Failed updating db for prekey"); + + debug("Starting new session for decryption with device from %s/%d", from_jid.to_string(), sid); + SessionCipher cipher = store.create_session_cipher(address); + key = cipher.decrypt_pre_key_message(msg); + // TODO: Finish session + } else { + debug("Continuing session for decryption with device from %s/%d", from_jid.to_string(), sid); + OmemoMessage msg = Plugin.get_context().deserialize_signal_message(encrypted_key); + SessionCipher cipher = store.create_session_cipher(address); + key = cipher.decrypt_message(msg); + } + + if (key.length >= 32) { + int authtaglength = key.length - 16; + uint8[] new_ciphertext = new uint8[ciphertext.length + authtaglength]; + uint8[] new_key = new uint8[16]; + Memory.copy(new_ciphertext, ciphertext, ciphertext.length); + Memory.copy((uint8*)new_ciphertext + ciphertext.length, (uint8*)key + 16, authtaglength); + Memory.copy(new_key, key, 16); + data.ciphertext = new_ciphertext; + key = new_key; + } + + return key; + } + + public override string decrypt(uint8[] ciphertext, uint8[] key, uint8[] iv) throws GLib.Error { + return arr_to_str(aes_decrypt(Cipher.AES_GCM_NOPADDING, key, iv, ciphertext)); + } + + private bool update_db_for_prekey(int identity_id, string identity_key, Jid from_jid, int sid) { + Row? device = db.identity_meta.get_device(identity_id, from_jid.to_string(), sid); + if (device != null && device[db.identity_meta.identity_key_public_base64] != null) { + if (device[db.identity_meta.identity_key_public_base64] != identity_key) { + critical("Tried to use a different identity key for a known device id."); + return false; + } + } else { + debug("Learn new device from incoming message from %s/%d", from_jid.to_string(), sid); + bool blind_trust = db.trust.get_blind_trust(identity_id, from_jid.to_string(), true); + if (db.identity_meta.insert_device_session(identity_id, from_jid.to_string(), sid, identity_key, blind_trust ? TrustLevel.TRUSTED : TrustLevel.UNKNOWN) < 0) { + critical("Failed learning a device."); + return false; + } + + XmppStream? stream = stream_interactor.get_stream(account); + if (device == null && stream != null) { + stream.get_module(StreamModule.IDENTITY).request_user_devicelist.begin(stream, from_jid); + } + } + return true; + } + + private string arr_to_str(uint8[] arr) { + // null-terminate the array + uint8[] rarr = new uint8[arr.length+1]; + Memory.copy(rarr, arr, arr.length); + return (string)rarr; + } + } + + public class DecryptMessageListener : MessageListener { + public string[] after_actions_const = new string[]{ }; + public override string action_group { get { return "DECRYPT"; } } + public override string[] after_actions { get { return after_actions_const; } } + + private HashMap<Account, OmemoDecryptor> decryptors; + + public DecryptMessageListener(HashMap<Account, OmemoDecryptor> decryptors) { + this.decryptors = decryptors; + } + + public override async bool run(Entities.Message message, Xmpp.MessageStanza stanza, Conversation conversation) { + decryptors[message.account].decrypt_message(message, stanza, conversation); + return false; + } + } +} + diff --git a/main/src/plugins/omemo/logic/encrypt.vala b/main/src/plugins/omemo/logic/encrypt.vala new file mode 100644 index 00000000..8994d00d --- /dev/null +++ b/main/src/plugins/omemo/logic/encrypt.vala @@ -0,0 +1,132 @@ +using Gee; +using Omemo; +using Dino.Crypto.Omemo; +using Dino.Entities; +using Xmpp; +using Xmpp.Xep.Omemo; + +namespace Dino.Plugins.Omemo { + + public class OmemoEncryptor : Xep.Omemo.OmemoEncryptor { + + private Account account; + private Store store; + private TrustManager trust_manager; + + public override uint32 own_device_id { get { return store.local_registration_id; }} + + public OmemoEncryptor(Account account, TrustManager trust_manager, Store store) { + this.account = account; + this.trust_manager = trust_manager; + this.store = store; + } + + public override Xep.Omemo.EncryptionData encrypt_plaintext(string plaintext) throws GLib.Error { + const uint KEY_SIZE = 16; + const uint IV_SIZE = 12; + + //Create a key and use it to encrypt the message + uint8[] key = new uint8[KEY_SIZE]; + Crypto.randomize(key); + uint8[] iv = new uint8[IV_SIZE]; + Crypto.randomize(iv); + + uint8[] aes_encrypt_result = aes_encrypt(Cipher.AES_GCM_NOPADDING, key, iv, plaintext.data); + uint8[] ciphertext = aes_encrypt_result[0:aes_encrypt_result.length - 16]; + uint8[] tag = aes_encrypt_result[aes_encrypt_result.length - 16:aes_encrypt_result.length]; + uint8[] keytag = new uint8[key.length + tag.length]; + Memory.copy(keytag, key, key.length); + Memory.copy((uint8*)keytag + key.length, tag, tag.length); + + var ret = new Xep.Omemo.EncryptionData(own_device_id); + ret.ciphertext = ciphertext; + ret.keytag = keytag; + ret.iv = iv; + return ret; + } + + public EncryptState encrypt(MessageStanza message, Jid self_jid, Gee.List<Jid> recipients, XmppStream stream) { + + EncryptState status = new EncryptState(); + if (!Plugin.ensure_context()) return status; + if (message.to == null) return status; + + try { + EncryptionData enc_data = encrypt_plaintext(message.body); + status = encrypt_key_to_recipients(enc_data, self_jid, recipients, stream); + + message.stanza.put_node(enc_data.get_encrypted_node()); + Xep.ExplicitEncryption.add_encryption_tag_to_message(message, NS_URI, "OMEMO"); + message.body = "[This message is OMEMO encrypted]"; + status.encrypted = true; + } catch (Error e) { + warning(@"error while encrypting message: $(e.message)\n"); + message.body = "[OMEMO encryption failed]"; + status.encrypted = false; + } + return status; + } + + internal EncryptState encrypt_key_to_recipients(EncryptionData enc_data, Jid self_jid, Gee.List<Jid> recipients, XmppStream stream) throws Error { + EncryptState status = new EncryptState(); + + //Check we have the bundles and device lists needed to send the message + if (!trust_manager.is_known_address(account, self_jid)) return status; + status.own_list = true; + status.own_devices = trust_manager.get_trusted_devices(account, self_jid).size; + status.other_waiting_lists = 0; + status.other_devices = 0; + foreach (Jid recipient in recipients) { + if (!trust_manager.is_known_address(account, recipient)) { + status.other_waiting_lists++; + } + if (status.other_waiting_lists > 0) return status; + status.other_devices += trust_manager.get_trusted_devices(account, recipient).size; + } + if (status.own_devices == 0 || status.other_devices == 0) return status; + + + //Encrypt the key for each recipient's device individually + foreach (Jid recipient in recipients) { + EncryptionResult enc_res = encrypt_key_to_recipient(stream, enc_data, recipient); + status.add_result(enc_res, false); + } + + // Encrypt the key for each own device + EncryptionResult enc_res = encrypt_key_to_recipient(stream, enc_data, self_jid); + status.add_result(enc_res, true); + + return status; + } + + public override EncryptionResult encrypt_key_to_recipient(XmppStream stream, Xep.Omemo.EncryptionData enc_data, Jid recipient) throws GLib.Error { + var result = new EncryptionResult(); + StreamModule module = stream.get_module(StreamModule.IDENTITY); + + foreach(int32 device_id in trust_manager.get_trusted_devices(account, recipient)) { + if (module.is_ignored_device(recipient, device_id)) { + result.lost++; + continue; + } + try { + encrypt_key(enc_data, recipient, device_id); + result.success++; + } catch (Error e) { + if (e.code == ErrorCode.UNKNOWN) result.unknown++; + else result.failure++; + } + } + return result; + } + + public override void encrypt_key(Xep.Omemo.EncryptionData encryption_data, Jid jid, int32 device_id) throws GLib.Error { + Address address = new Address(jid.to_string(), device_id); + SessionCipher cipher = store.create_session_cipher(address); + CiphertextMessage device_key = cipher.encrypt(encryption_data.keytag); + address.device_id = 0; + debug("Created encrypted key for %s/%d", jid.to_string(), device_id); + + encryption_data.add_device_key(device_id, device_key.serialized, device_key.type == CiphertextType.PREKEY); + } + } +} diff --git a/main/src/plugins/omemo/logic/manager.vala b/main/src/plugins/omemo/logic/manager.vala new file mode 100644 index 00000000..358db4e3 --- /dev/null +++ b/main/src/plugins/omemo/logic/manager.vala @@ -0,0 +1,431 @@ +using Gee; + +using Dino.Crypto.Omemo; +using Dino.Entities; +using Dino.Qlite; +using Omemo; +using Xmpp; + +namespace Dino.Plugins.Omemo { + +public class Manager : StreamInteractionModule, Object { + public static ModuleIdentity<Manager> IDENTITY = new ModuleIdentity<Manager>("omemo_manager"); + public string id { get { return IDENTITY.id; } } + + private StreamInteractor stream_interactor; + private Database db; + private TrustManager trust_manager; + private HashMap<Account, OmemoEncryptor> encryptors; + private Map<Entities.Message, MessageState> message_states = new HashMap<Entities.Message, MessageState>(Entities.Message.hash_func, Entities.Message.equals_func); + + private class MessageState { + public Entities.Message msg { get; private set; } + public Xep.Omemo.EncryptState last_try { get; private set; } + public int waiting_other_sessions { get; set; } + public int waiting_own_sessions { get; set; } + public bool waiting_own_devicelist { get; set; } + public int waiting_other_devicelists { get; set; } + public bool force_next_attempt { get; set; } + public bool will_send_now { get; private set; } + public bool active_send_attempt { get; set; } + + public MessageState(Entities.Message msg, Xep.Omemo.EncryptState last_try) { + update_from_encrypt_status(msg, last_try); + } + + public void update_from_encrypt_status(Entities.Message msg, Xep.Omemo.EncryptState new_try) { + this.msg = msg; + this.last_try = new_try; + this.waiting_other_sessions = new_try.other_unknown; + this.waiting_own_sessions = new_try.own_unknown; + this.waiting_own_devicelist = !new_try.own_list; + this.waiting_other_devicelists = new_try.other_waiting_lists; + this.active_send_attempt = false; + will_send_now = false; + if (new_try.other_failure > 0 || (new_try.other_lost == new_try.other_devices && new_try.other_devices > 0)) { + msg.marked = Entities.Message.Marked.WONTSEND; + } else if (new_try.other_unknown > 0 || new_try.own_unknown > 0 || new_try.other_waiting_lists > 0 || !new_try.own_list || new_try.own_devices == 0) { + msg.marked = Entities.Message.Marked.UNSENT; + } else if (!new_try.encrypted) { + msg.marked = Entities.Message.Marked.WONTSEND; + } else { + will_send_now = true; + } + } + + public bool should_retry_now() { + return !waiting_own_devicelist && waiting_other_devicelists <= 0 && waiting_other_sessions <= 0 && waiting_own_sessions <= 0 && !active_send_attempt; + } + + public string to_string() { + return @"MessageState (msg=$(msg.stanza_id), send=$will_send_now, $last_try)"; + } + } + + private Manager(StreamInteractor stream_interactor, Database db, TrustManager trust_manager, HashMap<Account, OmemoEncryptor> encryptors) { + this.stream_interactor = stream_interactor; + this.db = db; + this.trust_manager = trust_manager; + this.encryptors = encryptors; + + stream_interactor.account_added.connect(on_account_added); + stream_interactor.stream_negotiated.connect(on_stream_negotiated); + stream_interactor.get_module(MessageProcessor.IDENTITY).pre_message_send.connect(on_pre_message_send); + stream_interactor.get_module(RosterManager.IDENTITY).mutual_subscription.connect(on_mutual_subscription); + } + + public void clear_device_list(Account account) { + XmppStream? stream = stream_interactor.get_stream(account); + if (stream == null) return; + + stream.get_module(StreamModule.IDENTITY).clear_device_list(stream); + } + + private Gee.List<Jid> get_occupants(Jid jid, Account account){ + Gee.List<Jid> occupants = new ArrayList<Jid>(Jid.equals_bare_func); + if(!stream_interactor.get_module(MucManager.IDENTITY).is_groupchat(jid, account)){ + occupants.add(jid); + } + Gee.List<Jid>? occupant_jids = stream_interactor.get_module(MucManager.IDENTITY).get_offline_members(jid, account); + if(occupant_jids == null) { + return occupants; + } + foreach (Jid occupant in occupant_jids) { + if(!occupant.equals(account.bare_jid)){ + occupants.add(occupant.bare_jid); + } + } + return occupants; + } + + private void on_pre_message_send(Entities.Message message, Xmpp.MessageStanza message_stanza, Conversation conversation) { + if (message.encryption == Encryption.OMEMO) { + if (message.type_ == Message.Type.GROUPCHAT_PM) { + message.marked = Message.Marked.WONTSEND; + return; + } + XmppStream? stream = stream_interactor.get_stream(conversation.account); + if (stream == null) { + message.marked = Entities.Message.Marked.UNSENT; + return; + } + StreamModule? module_ = ((!)stream).get_module(StreamModule.IDENTITY); + if (module_ == null) { + message.marked = Entities.Message.Marked.UNSENT; + return; + } + StreamModule module = (!)module_; + + //Get a list of everyone for whom the message should be encrypted + Gee.List<Jid> recipients; + if (message_stanza.type_ == MessageStanza.TYPE_GROUPCHAT) { + recipients = get_occupants((!)message.to.bare_jid, conversation.account); + if (recipients.size == 0) { + message.marked = Entities.Message.Marked.WONTSEND; + return; + } + } else { + recipients = new ArrayList<Jid>(Jid.equals_bare_func); + recipients.add(message_stanza.to); + } + + //Attempt to encrypt the message + Xep.Omemo.EncryptState enc_state = encryptors[conversation.account].encrypt(message_stanza, conversation.account.bare_jid, recipients, stream); + MessageState state; + lock (message_states) { + if (message_states.has_key(message)) { + state = message_states.get(message); + state.update_from_encrypt_status(message, enc_state); + if (state.will_send_now) { + debug("sending message delayed: %s", state.to_string()); + } + } else { + state = new MessageState(message, enc_state); + message_states[message] = state; + } + if (state.will_send_now) { + message_states.unset(message); + } + } + + //Encryption failed - need to fetch more information + if (!state.will_send_now) { + if (message.marked == Entities.Message.Marked.WONTSEND) { + debug("retracting message %s", state.to_string()); + message_states.unset(message); + } else { + debug("delaying message %s", state.to_string()); + + if (state.waiting_own_sessions > 0) { + module.fetch_bundles((!)stream, conversation.account.bare_jid, trust_manager.get_trusted_devices(conversation.account, conversation.account.bare_jid)); + } + if (state.waiting_other_sessions > 0 && message.counterpart != null) { + foreach(Jid jid in get_occupants(((!)message.counterpart).bare_jid, conversation.account)) { + module.fetch_bundles((!)stream, jid, trust_manager.get_trusted_devices(conversation.account, jid)); + } + } + if (state.waiting_other_devicelists > 0 && message.counterpart != null) { + foreach(Jid jid in get_occupants(((!)message.counterpart).bare_jid, conversation.account)) { + module.request_user_devicelist.begin((!)stream, jid); + } + } + } + } + } + } + + private void on_mutual_subscription(Account account, Jid jid) { + XmppStream? stream = stream_interactor.get_stream(account); + if(stream == null) return; + + stream_interactor.module_manager.get_module(account, StreamModule.IDENTITY).request_user_devicelist.begin((!)stream, jid); + } + + private void on_stream_negotiated(Account account, XmppStream stream) { + StreamModule module = stream_interactor.module_manager.get_module(account, StreamModule.IDENTITY); + if (module != null) { + module.request_user_devicelist.begin(stream, account.bare_jid); + } + } + + private void on_account_added(Account account) { + StreamModule module = stream_interactor.module_manager.get_module(account, StreamModule.IDENTITY); + if (module != null) { + module.device_list_loaded.connect((jid, devices) => on_device_list_loaded(account, jid, devices)); + module.bundle_fetched.connect((jid, device_id, bundle) => on_bundle_fetched(account, jid, device_id, bundle)); + module.bundle_fetch_failed.connect((jid) => continue_message_sending(account, jid)); + } + initialize_store.begin(account); + } + + private void on_device_list_loaded(Account account, Jid jid, ArrayList<int32> device_list) { + debug("received device list for %s from %s", account.bare_jid.to_string(), jid.to_string()); + + XmppStream? stream = stream_interactor.get_stream(account); + if (stream == null) { + return; + } + StreamModule? module = ((!)stream).get_module(StreamModule.IDENTITY); + if (module == null) { + return; + } + + int identity_id = db.identity.get_id(account.id); + if (identity_id < 0) return; + + //Update meta database + db.identity_meta.insert_device_list(identity_id, jid.bare_jid.to_string(), device_list); + + //Fetch the bundle for each new device + int inc = 0; + foreach (Row row in db.identity_meta.get_unknown_devices(identity_id, jid.bare_jid.to_string())) { + try { + module.fetch_bundle(stream, Jid.from_string(row[db.identity_meta.address_name]), row[db.identity_meta.device_id], false); + inc++; + } catch (InvalidJidError e) { + warning("Ignoring device with invalid Jid: %s", e.message); + } + } + if (inc > 0) { + debug("new bundles %i/%i for %s", inc, device_list.size, jid.to_string()); + } + + //Create an entry for the jid in the account table if one does not exist already + if (db.trust.select().with(db.trust.identity_id, "=", identity_id).with(db.trust.address_name, "=", jid.bare_jid.to_string()).count() == 0) { + db.trust.insert().value(db.trust.identity_id, identity_id).value(db.trust.address_name, jid.bare_jid.to_string()).value(db.trust.blind_trust, true).perform(); + } + + //Get all messages that needed the devicelist and determine if we can now send them + HashSet<Entities.Message> send_now = new HashSet<Entities.Message>(); + lock (message_states) { + foreach (Entities.Message msg in message_states.keys) { + if (!msg.account.equals(account)) continue; + Gee.List<Jid> occupants = get_occupants(msg.counterpart.bare_jid, account); + MessageState state = message_states[msg]; + if (account.bare_jid.equals(jid)) { + state.waiting_own_devicelist = false; + } else if (msg.counterpart != null && occupants.contains(jid)) { + state.waiting_other_devicelists--; + } + if (state.should_retry_now()) { + send_now.add(msg); + state.active_send_attempt = true; + } + } + } + foreach (Entities.Message msg in send_now) { + if (msg.counterpart == null) continue; + Entities.Conversation? conv = stream_interactor.get_module(ConversationManager.IDENTITY).get_conversation(((!)msg.counterpart), account); + if (conv == null) continue; + stream_interactor.get_module(MessageProcessor.IDENTITY).send_xmpp_message(msg, (!)conv, true); + } + + } + + private void on_bundle_fetched(Account account, Jid jid, int32 device_id, Bundle bundle) { + int identity_id = db.identity.get_id(account.id); + if (identity_id < 0) return; + + bool blind_trust = db.trust.get_blind_trust(identity_id, jid.bare_jid.to_string(), true); + + //If we don't blindly trust new devices and we haven't seen this key before then don't trust it + bool untrust = !(blind_trust || db.identity_meta.with_address(identity_id, jid.bare_jid.to_string()) + .with(db.identity_meta.device_id, "=", device_id) + .with(db.identity_meta.identity_key_public_base64, "=", Base64.encode(bundle.identity_key.serialize())) + .single().row().is_present()); + + //Get trust information from the database if the device id is known + Row device = db.identity_meta.get_device(identity_id, jid.bare_jid.to_string(), device_id); + TrustLevel trusted = TrustLevel.UNKNOWN; + if (device != null) { + trusted = (TrustLevel) device[db.identity_meta.trust_level]; + } + + if(untrust) { + trusted = TrustLevel.UNKNOWN; + } else if (blind_trust && trusted == TrustLevel.UNKNOWN) { + trusted = TrustLevel.TRUSTED; + } + + //Update the database with the appropriate trust information + db.identity_meta.insert_device_bundle(identity_id, jid.bare_jid.to_string(), device_id, bundle, trusted); + + if (should_start_session(account, jid)) { + XmppStream? stream = stream_interactor.get_stream(account); + if (stream != null) { + StreamModule? module = ((!)stream).get_module(StreamModule.IDENTITY); + if (module != null) { + module.start_session(stream, jid, device_id, bundle); + } + } + } + continue_message_sending(account, jid); + } + + private bool should_start_session(Account account, Jid jid) { + lock (message_states) { + foreach (Entities.Message msg in message_states.keys) { + if (!msg.account.equals(account)) continue; + Gee.List<Jid> occupants = get_occupants(msg.counterpart.bare_jid, account); + if (account.bare_jid.equals(jid) || (msg.counterpart != null && (msg.counterpart.equals_bare(jid) || occupants.contains(jid)))) { + return true; + } + } + } + return false; + } + + private void continue_message_sending(Account account, Jid jid) { + //Get all messages waiting and determine if they can now be sent + HashSet<Entities.Message> send_now = new HashSet<Entities.Message>(); + lock (message_states) { + foreach (Entities.Message msg in message_states.keys) { + if (!msg.account.equals(account)) continue; + Gee.List<Jid> occupants = get_occupants(msg.counterpart.bare_jid, account); + + MessageState state = message_states[msg]; + + if (account.bare_jid.equals(jid)) { + state.waiting_own_sessions--; + } else if (msg.counterpart != null && (msg.counterpart.equals_bare(jid) || occupants.contains(jid))) { + state.waiting_other_sessions--; + } + if (state.should_retry_now()){ + send_now.add(msg); + state.active_send_attempt = true; + } + } + } + foreach (Entities.Message msg in send_now) { + if (msg.counterpart == null) continue; + Entities.Conversation? conv = stream_interactor.get_module(ConversationManager.IDENTITY).get_conversation((!)msg.counterpart, account); + if (conv == null) continue; + stream_interactor.get_module(MessageProcessor.IDENTITY).send_xmpp_message(msg, (!)conv, true); + } + } + + private async void initialize_store(Account account) { + // If the account is not yet persisted, wait for that and then continue - without identity.account_id the entry isn't worth much. + if (account.id == -1) { + account.notify["id"].connect(() => initialize_store.callback()); + yield; + } + StreamModule? module = stream_interactor.module_manager.get_module(account, StreamModule.IDENTITY); + if (module == null) return; + Store store = module.store; + Qlite.Row? row = db.identity.row_with(db.identity.account_id, account.id).inner; + int identity_id = -1; + bool publish_identity = false; + + if (row == null) { + // OMEMO not yet initialized, starting with empty base + publish_identity = true; + try { + store.identity_key_store.local_registration_id = Random.int_range(1, int32.MAX); + + ECKeyPair key_pair = Plugin.get_context().generate_key_pair(); + store.identity_key_store.identity_key_private = new Bytes(key_pair.private.serialize()); + store.identity_key_store.identity_key_public = new Bytes(key_pair.public.serialize()); + + identity_id = (int) db.identity.upsert() + .value(db.identity.account_id, account.id, true) + .value(db.identity.device_id, (int) store.local_registration_id) + .value(db.identity.identity_key_private_base64, Base64.encode(store.identity_key_store.identity_key_private.get_data())) + .value(db.identity.identity_key_public_base64, Base64.encode(store.identity_key_store.identity_key_public.get_data())) + .perform(); + } catch (Error e) { + // Ignore error + } + } else { + store.identity_key_store.local_registration_id = ((!)row)[db.identity.device_id]; + store.identity_key_store.identity_key_private = new Bytes(Base64.decode(((!)row)[db.identity.identity_key_private_base64])); + store.identity_key_store.identity_key_public = new Bytes(Base64.decode(((!)row)[db.identity.identity_key_public_base64])); + identity_id = ((!)row)[db.identity.id]; + } + + if (identity_id >= 0) { + store.signed_pre_key_store = new BackedSignedPreKeyStore(db, identity_id); + store.pre_key_store = new BackedPreKeyStore(db, identity_id); + store.session_store = new BackedSessionStore(db, identity_id); + } else { + warning("store for %s is not persisted!", account.bare_jid.to_string()); + } + + // Generated new device ID, ensure this gets added to the devicelist + XmppStream? stream = stream_interactor.get_stream(account); + if (stream != null) { + module.request_user_devicelist.begin((!)stream, account.bare_jid); + } + } + + public async bool ensure_get_keys_for_conversation(Conversation conversation) { + if (stream_interactor.get_module(MucManager.IDENTITY).is_private_room(conversation.account, conversation.counterpart)) { + foreach (Jid offline_member in stream_interactor.get_module(MucManager.IDENTITY).get_offline_members(conversation.counterpart, conversation.account)) { + bool ok = yield ensure_get_keys_for_jid(conversation.account, offline_member); + if (!ok) { + return false; + } + } + return true; + } + + return yield ensure_get_keys_for_jid(conversation.account, conversation.counterpart.bare_jid); + } + + public async bool ensure_get_keys_for_jid(Account account, Jid jid) { + if (trust_manager.is_known_address(account, jid)) return true; + XmppStream? stream = stream_interactor.get_stream(account); + if (stream != null) { + var device_list = yield stream_interactor.module_manager.get_module(account, StreamModule.IDENTITY).request_user_devicelist(stream, jid); + return device_list.size > 0; + } + return true; // TODO wait for stream? + } + + public static void start(StreamInteractor stream_interactor, Database db, TrustManager trust_manager, HashMap<Account, OmemoEncryptor> encryptors) { + Manager m = new Manager(stream_interactor, db, trust_manager, encryptors); + stream_interactor.add_module(m); + } +} + +} diff --git a/main/src/plugins/omemo/logic/pre_key_store.vala b/main/src/plugins/omemo/logic/pre_key_store.vala new file mode 100644 index 00000000..97e27858 --- /dev/null +++ b/main/src/plugins/omemo/logic/pre_key_store.vala @@ -0,0 +1,48 @@ +using Dino.Crypto.Omemo; +using Dino.Qlite; +using Omemo; + +namespace Dino.Plugins.Omemo { + +private class BackedPreKeyStore : PreKeyStore { + private Database db; + private int identity_id; + + public BackedPreKeyStore(Database db, int identity_id) { + this.db = db; + this.identity_id = identity_id; + } + + public override uint8[]? load_pre_key(uint32 pre_key_id) throws Crypto.Error { + RowOption row = db.pre_key.select({db.pre_key.record_base64}) + .with(db.pre_key.identity_id, "=", identity_id) + .with(db.pre_key.pre_key_id, "=", (int) pre_key_id) + .single().row(); + if (!row.is_present()) return null; + return Base64.decode(row[db.pre_key.record_base64]); + } + + public override void store_pre_key(uint32 pre_key_id, uint8[] record) throws Crypto.Error { + db.pre_key.upsert() + .value(db.pre_key.identity_id, identity_id, true) + .value(db.pre_key.pre_key_id, (int) pre_key_id, true) + .value(db.pre_key.record_base64, Base64.encode(record)) + .perform(); + } + + public override bool contains_pre_key(uint32 pre_key_id) throws Crypto.Error { + return db.pre_key.select({db.pre_key.pre_key_id}) + .with(db.pre_key.identity_id, "=", identity_id) + .with(db.pre_key.pre_key_id, "=", (int) pre_key_id) + .count() > 0; + } + + public override void delete_pre_key(uint32 pre_key_id) throws Crypto.Error { + db.pre_key.delete() + .with(db.pre_key.identity_id, "=", identity_id) + .with(db.pre_key.pre_key_id, "=", (int) pre_key_id) + .perform(); + } +} + +} diff --git a/main/src/plugins/omemo/logic/session_store.vala b/main/src/plugins/omemo/logic/session_store.vala new file mode 100644 index 00000000..883e1caa --- /dev/null +++ b/main/src/plugins/omemo/logic/session_store.vala @@ -0,0 +1,69 @@ +using Dino.Crypto.Omemo; +using Dino.Qlite; +using Omemo; + +namespace Dino.Plugins.Omemo { + +private class BackedSessionStore : SessionStore { + private Database db; + private int identity_id; + + public BackedSessionStore(Database db, int identity_id) { + this.db = db; + this.identity_id = identity_id; + } + + public override uint8[]? load_session(Address address) throws Crypto.Error { + RowOption row = db.session.select({db.session.record_base64}) + .with(db.session.identity_id, "=", identity_id) + .with(db.session.address_name, "=", address.name) + .with(db.session.device_id, "=", address.device_id) + .single().row(); + if (!row.is_present()) return null; + return Base64.decode(row[db.session.record_base64]); + } + + public override IntList get_sub_device_sessions(string name) throws Crypto.Error { + IntList res = new IntList(); + foreach (Row row in db.session.select({db.session.device_id}) + .with(db.session.identity_id, "=", identity_id) + .with(db.session.address_name, "=", name)) { + res.add(row[db.session.device_id]); + } + return res; + } + + public override void store_session(Address address, uint8[] record) throws Crypto.Error { + db.session.upsert() + .value(db.session.identity_id, identity_id, true) + .value(db.session.address_name, address.name, true) + .value(db.session.device_id, address.device_id, true) + .value(db.session.record_base64, Base64.encode(record)) + .perform(); + } + + public override bool contains_session(Address address) throws Crypto.Error { + return db.session.select({db.session.device_id}) + .with(db.session.identity_id, "=", identity_id) + .with(db.session.address_name, "=", address.name) + .with(db.session.device_id, "=", address.device_id) + .count() > 0; + } + + public override void delete_session(Address address) throws Crypto.Error { + db.session.delete() + .with(db.session.identity_id, "=", identity_id) + .with(db.session.address_name, "=", address.name) + .with(db.session.device_id, "=", address.device_id) + .perform(); + } + + public override void delete_all_sessions(string name) throws Crypto.Error { + db.session.delete() + .with(db.session.identity_id, "=", identity_id) + .with(db.session.address_name, "=", name) + .perform(); + } +} + +} diff --git a/main/src/plugins/omemo/logic/signed_pre_key_store.vala b/main/src/plugins/omemo/logic/signed_pre_key_store.vala new file mode 100644 index 00000000..f4d57550 --- /dev/null +++ b/main/src/plugins/omemo/logic/signed_pre_key_store.vala @@ -0,0 +1,48 @@ +using Dino.Crypto.Omemo; +using Dino.Qlite; +using Omemo; + +namespace Dino.Plugins.Omemo { + +private class BackedSignedPreKeyStore : SignedPreKeyStore { + private Database db; + private int identity_id; + + public BackedSignedPreKeyStore(Database db, int identity_id) { + this.db = db; + this.identity_id = identity_id; + } + + public override uint8[]? load_signed_pre_key(uint32 pre_key_id) throws Crypto.Error { + RowOption row = db.signed_pre_key.select({db.signed_pre_key.record_base64}) + .with(db.signed_pre_key.identity_id, "=", identity_id) + .with(db.signed_pre_key.signed_pre_key_id, "=", (int) pre_key_id) + .single().row(); + if (!row.is_present()) return null; + return Base64.decode(row[db.signed_pre_key.record_base64]); + } + + public override void store_signed_pre_key(uint32 pre_key_id, uint8[] record) throws Crypto.Error { + db.signed_pre_key.upsert() + .value(db.signed_pre_key.identity_id, identity_id, true) + .value(db.signed_pre_key.signed_pre_key_id, (int) pre_key_id, true) + .value(db.signed_pre_key.record_base64, Base64.encode(record)) + .perform(); + } + + public override bool contains_signed_pre_key(uint32 pre_key_id) throws Crypto.Error { + return db.signed_pre_key.select({db.signed_pre_key.signed_pre_key_id}) + .with(db.signed_pre_key.identity_id, "=", identity_id) + .with(db.signed_pre_key.signed_pre_key_id, "=", (int) pre_key_id) + .count() > 0; + } + + public override void delete_signed_pre_key(uint32 pre_key_id) throws Crypto.Error { + db.signed_pre_key.delete() + .with(db.signed_pre_key.identity_id, "=", identity_id) + .with(db.signed_pre_key.signed_pre_key_id, "=", (int) pre_key_id) + .perform(); + } +} + +} diff --git a/main/src/plugins/omemo/logic/trust_manager.vala b/main/src/plugins/omemo/logic/trust_manager.vala new file mode 100644 index 00000000..f17ade99 --- /dev/null +++ b/main/src/plugins/omemo/logic/trust_manager.vala @@ -0,0 +1,142 @@ +using Gee; + +using Dino.Entities; +using Dino.Qlite; +using Omemo; +using Xmpp; + +namespace Dino.Plugins.Omemo { + +public class TrustManager { + + public signal void bad_message_state_updated(Account account, Jid jid, int device_id); + + private StreamInteractor stream_interactor; + private Database db; + private TagMessageListener tag_message_listener; + + public HashMap<Message, int> message_device_id_map = new HashMap<Message, int>(Message.hash_func, Message.equals_func); + + public TrustManager(StreamInteractor stream_interactor, Database db) { + this.stream_interactor = stream_interactor; + this.db = db; + + tag_message_listener = new TagMessageListener(stream_interactor, this, db, message_device_id_map); + stream_interactor.get_module(MessageProcessor.IDENTITY).received_pipeline.connect(tag_message_listener); + } + + public void set_blind_trust(Account account, Jid jid, bool blind_trust) { + int identity_id = db.identity.get_id(account.id); + if (identity_id < 0) return; + db.trust.update() + .with(db.trust.identity_id, "=", identity_id) + .with(db.trust.address_name, "=", jid.bare_jid.to_string()) + .set(db.trust.blind_trust, blind_trust).perform(); + } + + public void set_device_trust(Account account, Jid jid, int device_id, TrustLevel trust_level) { + int identity_id = db.identity.get_id(account.id); + db.identity_meta.update() + .with(db.identity_meta.identity_id, "=", identity_id) + .with(db.identity_meta.address_name, "=", jid.bare_jid.to_string()) + .with(db.identity_meta.device_id, "=", device_id) + .set(db.identity_meta.trust_level, trust_level).perform(); + + // Hide messages from untrusted or unknown devices + string selection = null; + string[] selection_args = {}; + var app_db = Application.get_default().db; + foreach (Row row in db.content_item_meta.with_device(identity_id, jid.bare_jid.to_string(), device_id).with(db.content_item_meta.trusted_when_received, "=", false)) { + if (selection == null) { + selection = @"$(app_db.content_item.id) = ?"; + } else { + selection += @" OR $(app_db.content_item.id) = ?"; + } + selection_args += row[db.content_item_meta.content_item_id].to_string(); + } + if (selection != null) { + app_db.content_item.update() + .set(app_db.content_item.hide, trust_level == TrustLevel.UNTRUSTED || trust_level == TrustLevel.UNKNOWN) + .where(selection, selection_args) + .perform(); + } + + if (trust_level == TrustLevel.TRUSTED) { + db.identity_meta.update_last_message_untrusted(identity_id, device_id, null); + bad_message_state_updated(account, jid, device_id); + } + } + + public bool is_known_address(Account account, Jid jid) { + int identity_id = db.identity.get_id(account.id); + if (identity_id < 0) return false; + return db.identity_meta.with_address(identity_id, jid.to_string()).with(db.identity_meta.last_active, ">", 0).count() > 0; + } + + public Gee.List<int32> get_trusted_devices(Account account, Jid jid) { + Gee.List<int32> devices = new ArrayList<int32>(); + int identity_id = db.identity.get_id(account.id); + if (identity_id < 0) return devices; + foreach (Row device in db.identity_meta.get_trusted_devices(identity_id, jid.bare_jid.to_string())) { + if(device[db.identity_meta.trust_level] != TrustLevel.UNKNOWN || device[db.identity_meta.identity_key_public_base64] == null) + devices.add(device[db.identity_meta.device_id]); + } + return devices; + } + + private class TagMessageListener : MessageListener { + public string[] after_actions_const = new string[]{ "STORE" }; + public override string action_group { get { return "DECRYPT_TAG"; } } + public override string[] after_actions { get { return after_actions_const; } } + + private StreamInteractor stream_interactor; + private TrustManager trust_manager; + private Database db; + private HashMap<Message, int> message_device_id_map; + + public TagMessageListener(StreamInteractor stream_interactor, TrustManager trust_manager, Database db, HashMap<Message, int> message_device_id_map) { + this.stream_interactor = stream_interactor; + this.trust_manager = trust_manager; + this.db = db; + this.message_device_id_map = message_device_id_map; + } + + public override async bool run(Entities.Message message, Xmpp.MessageStanza stanza, Conversation conversation) { + int device_id = 0; + if (message_device_id_map.has_key(message)) { + device_id = message_device_id_map[message]; + message_device_id_map.unset(message); + } + + // TODO: Handling of files + + ContentItem? content_item = stream_interactor.get_module(ContentItemStore.IDENTITY).get_item_by_foreign(conversation, 1, message.id); + + if (content_item != null && device_id != 0) { + Jid jid = content_item.jid; + if (conversation.type_ == Conversation.Type.GROUPCHAT) { + jid = message.real_jid; + } + + int identity_id = db.identity.get_id(conversation.account.id); + TrustLevel trust_level = (TrustLevel) db.identity_meta.get_device(identity_id, jid.bare_jid.to_string(), device_id)[db.identity_meta.trust_level]; + if (trust_level == TrustLevel.UNTRUSTED || trust_level == TrustLevel.UNKNOWN) { + stream_interactor.get_module(ContentItemStore.IDENTITY).set_item_hide(content_item, true); + db.identity_meta.update_last_message_untrusted(identity_id, device_id, message.time); + trust_manager.bad_message_state_updated(conversation.account, jid, device_id); + } + + db.content_item_meta.insert() + .value(db.content_item_meta.content_item_id, content_item.id) + .value(db.content_item_meta.identity_id, identity_id) + .value(db.content_item_meta.address_name, jid.bare_jid.to_string()) + .value(db.content_item_meta.device_id, device_id) + .value(db.content_item_meta.trusted_when_received, trust_level != TrustLevel.UNTRUSTED) + .perform(); + } + return false; + } + } +} + +} diff --git a/main/src/plugins/omemo/plugin.vala b/main/src/plugins/omemo/plugin.vala new file mode 100644 index 00000000..34528bde --- /dev/null +++ b/main/src/plugins/omemo/plugin.vala @@ -0,0 +1,87 @@ +using Gee; +using Dino.Crypto.Omemo; +using Dino.Entities; +using Omemo; + +namespace Dino.Plugins.Omemo { + +public class Plugin : RootInterface, Object { + public const bool DEBUG = false; + private static Context? _context; + public static Context get_context() { + assert(_context != null); + return (!)_context; + } + public static bool ensure_context() { + lock(_context) { + try { + if (_context == null) { + _context = new Context(DEBUG); + } + return true; + } catch (Error e) { + warning("Error initializing libomemo-c Context %s", e.message); + return false; + } + } + } + + public Dino.Application app; + public Database db; + public EncryptionListEntry list_entry; + public ContactDetailsProvider contact_details_provider; + public DeviceNotificationPopulator device_notification_populator; + public OwnNotifications own_notifications; + public TrustManager trust_manager; + public HashMap<Account, OmemoDecryptor> decryptors = new HashMap<Account, OmemoDecryptor>(Account.hash_func, Account.equals_func); + public HashMap<Account, OmemoEncryptor> encryptors = new HashMap<Account, OmemoEncryptor>(Account.hash_func, Account.equals_func); + + public void registered(Dino.Application app) { + ensure_context(); + this.app = app; + this.db = new Database(Path.build_filename(Application.get_storage_dir(), "omemo.db")); + this.list_entry = new EncryptionListEntry(this); + this.contact_details_provider = new ContactDetailsProvider(this); + this.device_notification_populator = new DeviceNotificationPopulator(this, this.app.stream_interactor); + this.trust_manager = new TrustManager(this.app.stream_interactor, this.db); + + this.app.plugin_registry.register_encryption_list_entry(list_entry); + this.app.plugin_registry.register_encryption_preferences_entry(new OmemoPreferencesEntry(this)); + this.app.plugin_registry.register_contact_details_entry(contact_details_provider); + this.app.plugin_registry.register_notification_populator(device_notification_populator); + this.app.plugin_registry.register_conversation_addition_populator(new BadMessagesPopulator(this.app.stream_interactor, this)); + this.app.plugin_registry.register_call_entryption_entry(DtlsSrtpVerificationDraft.NS_URI, new CallEncryptionEntry(db)); + + this.app.stream_interactor.module_manager.initialize_account_modules.connect((account, list) => { + Store store = Plugin.get_context().create_store(); + list.add(new StreamModule(store)); + decryptors[account] = new OmemoDecryptor(account, app.stream_interactor, trust_manager, db, store); + list.add(decryptors[account]); + encryptors[account] = new OmemoEncryptor(account, trust_manager, store); + list.add(encryptors[account]); + list.add(new JetOmemo.Module()); + list.add(new DtlsSrtpVerificationDraft.StreamModule()); + this.own_notifications = new OwnNotifications(this, this.app.stream_interactor, account); + }); + + app.stream_interactor.get_module(MessageProcessor.IDENTITY).received_pipeline.connect(new DecryptMessageListener(decryptors)); + app.stream_interactor.get_module(FileManager.IDENTITY).add_file_decryptor(new OmemoFileDecryptor()); + app.stream_interactor.get_module(FileManager.IDENTITY).add_file_encryptor(new OmemoFileEncryptor()); + JingleFileHelperRegistry.instance.add_encryption_helper(Encryption.OMEMO, new JetOmemo.EncryptionHelper(app.stream_interactor)); + + Manager.start(this.app.stream_interactor, db, trust_manager, encryptors); + } + + public void shutdown() { + // Nothing to do + } + + public bool has_new_devices(Account account, Xmpp.Jid jid) { + int identity_id = db.identity.get_id(account.id); + if (identity_id < 0) return false; + + return db.identity_meta.get_new_devices(identity_id, jid.bare_jid.to_string()).count() > 0; + } +} + +} diff --git a/main/src/plugins/omemo/protocol/bundle.vala b/main/src/plugins/omemo/protocol/bundle.vala new file mode 100644 index 00000000..abd5269f --- /dev/null +++ b/main/src/plugins/omemo/protocol/bundle.vala @@ -0,0 +1,92 @@ +using Gee; +using Omemo; +using Xmpp; + +namespace Dino.Plugins.Omemo { + +public class Bundle { + public StanzaNode? node; + + public Bundle(StanzaNode? node) { + this.node = node; + assert(Plugin.ensure_context()); + } + + public int32 signed_pre_key_id { get { + if (node == null) return -1; + string? id = ((!)node).get_deep_attribute("signedPreKeyPublic", "signedPreKeyId"); + if (id == null) return -1; + return int.parse((!)id); + }} + + public ECPublicKey? signed_pre_key { owned get { + if (node == null) return null; + string? key = ((!)node).get_deep_string_content("signedPreKeyPublic"); + if (key == null) return null; + try { + return Plugin.get_context().decode_public_key(Base64.decode((!)key)); + } catch (Error e) { + return null; + } + }} + + public uint8[]? signed_pre_key_signature { owned get { + if (node == null) return null; + string? sig = ((!)node).get_deep_string_content("signedPreKeySignature"); + if (sig == null) return null; + return Base64.decode((!)sig); + }} + + public ECPublicKey? identity_key { owned get { + if (node == null) return null; + string? key = ((!)node).get_deep_string_content("identityKey"); + if (key == null) return null; + try { + return Plugin.get_context().decode_public_key(Base64.decode((!)key)); + } catch (Error e) { + return null; + } + }} + + public ArrayList<PreKey> pre_keys { owned get { + ArrayList<PreKey> list = new ArrayList<PreKey>(); + if (node == null || ((!)node).get_subnode("prekeys") == null) return list; + ((!)node).get_deep_subnodes("prekeys", "preKeyPublic") + .filter((node) => ((!)node).get_attribute("preKeyId") != null) + .map<PreKey>(PreKey.create) + .foreach((key) => list.add(key)); + return list; + }} + + public bool is_valid { get { + return signed_pre_key_id != -1 && signed_pre_key != null && signed_pre_key_signature != null && identity_key != null; + }} + + public class PreKey { + private StanzaNode node; + + public static PreKey create(owned StanzaNode node) { + return new PreKey(node); + } + + public PreKey(StanzaNode node) { + this.node = node; + } + + public int32 key_id { get { + return int.parse(node.get_attribute("preKeyId") ?? "-1"); + }} + + public ECPublicKey? key { owned get { + string? key = node.get_string_content(); + if (key == null) return null; + try { + return Plugin.get_context().decode_public_key(Base64.decode((!)key)); + } catch (Error e) { + return null; + } + }} + } +} + +} diff --git a/main/src/plugins/omemo/protocol/message_flag.vala b/main/src/plugins/omemo/protocol/message_flag.vala new file mode 100644 index 00000000..ba9ea16e --- /dev/null +++ b/main/src/plugins/omemo/protocol/message_flag.vala @@ -0,0 +1,23 @@ +using Xmpp; + +namespace Dino.Plugins.Omemo { + +public class MessageFlag : Xmpp.MessageFlag { + public const string id = "omemo"; + + public bool decrypted = false; + + public static MessageFlag? get_flag(MessageStanza message) { + return (MessageFlag) message.get_flag(NS_URI, id); + } + + public override string get_ns() { + return NS_URI; + } + + public override string get_id() { + return id; + } +} + +}
\ No newline at end of file diff --git a/main/src/plugins/omemo/protocol/stream_module.vala b/main/src/plugins/omemo/protocol/stream_module.vala new file mode 100644 index 00000000..3958bc23 --- /dev/null +++ b/main/src/plugins/omemo/protocol/stream_module.vala @@ -0,0 +1,337 @@ +using Gee; +using Dino.Crypto.Omemo; +using Xmpp; +using Xmpp.Xep; +using Omemo; + +namespace Dino.Plugins.Omemo { + +internal const string NS_URI = "eu.siacs.conversations.axolotl"; +private const string NODE_DEVICELIST = NS_URI + ".devicelist"; +private const string NODE_BUNDLES = NS_URI + ".bundles"; +private const string NODE_VERIFICATION = NS_URI + ".verification"; + +private const int NUM_KEYS_TO_PUBLISH = 100; + +public class StreamModule : XmppStreamModule { + public static Xmpp.ModuleIdentity<StreamModule> IDENTITY = new Xmpp.ModuleIdentity<StreamModule>(NS_URI, "omemo_module"); + private static TimeSpan IGNORE_TIME = TimeSpan.MINUTE; + + public Store store { public get; private set; } + private ConcurrentSet<string> active_bundle_requests = new ConcurrentSet<string>(); + private HashMap<Jid, Future<ArrayList<int32>>> active_devicelist_requests = new HashMap<Jid, Future<ArrayList<int32>>>(Jid.hash_func, Jid.equals_func); + private Map<string, DateTime> device_ignore_time = new HashMap<string, DateTime>(); + + public signal void device_list_loaded(Jid jid, ArrayList<int32> devices); + public signal void bundle_fetched(Jid jid, int device_id, Bundle bundle); + public signal void bundle_fetch_failed(Jid jid, int device_id); + + public StreamModule(Store store) { + this.store = store; + } + + public override void attach(XmppStream stream) { + stream.get_module(Pubsub.Module.IDENTITY).add_filtered_notification(stream, NODE_DEVICELIST, + (stream, jid, id, node) => parse_device_list(stream, jid, id, node), null, null); + } + + public override void detach(XmppStream stream) { + stream.get_module(Pubsub.Module.IDENTITY).remove_filtered_notification(stream, NODE_DEVICELIST); + } + + public async ArrayList<int32> request_user_devicelist(XmppStream stream, Jid jid) { + var future = active_devicelist_requests[jid]; + if (future == null) { + var promise = new Promise<ArrayList<int32>?>(); + future = promise.future; + active_devicelist_requests[jid] = future; + + stream.get_module(Pubsub.Module.IDENTITY).request(stream, jid, NODE_DEVICELIST, (stream, jid, id, node) => { + ArrayList<int32> device_list = parse_device_list(stream, jid, id, node); + promise.set_value(device_list); + active_devicelist_requests.unset(jid); + }); + } + + try { + ArrayList<int32> device_list = yield future.wait_async(); + return device_list; + } catch (FutureError error) { + warning("Future error when waiting for device list: %s", error.message); + return new ArrayList<int32>(); + } + } + + public ArrayList<int32> parse_device_list(XmppStream stream, Jid jid, string? id, StanzaNode? node_) { + ArrayList<int32> device_list = new ArrayList<int32>(); + + StanzaNode node = node_ ?? new StanzaNode.build("list", NS_URI).add_self_xmlns(); + Jid? my_jid = stream.get_flag(Bind.Flag.IDENTITY).my_jid; + if (my_jid == null) return device_list; + if (jid.equals_bare(my_jid) && store.local_registration_id != 0) { + bool am_on_devicelist = false; + foreach (StanzaNode device_node in node.get_subnodes("device")) { + int device_id = device_node.get_attribute_int("id"); + if (store.local_registration_id == device_id) { + am_on_devicelist = true; + } + } + if (!am_on_devicelist) { + debug("Not on device list, adding id"); + node.put_node(new StanzaNode.build("device", NS_URI).put_attribute("id", store.local_registration_id.to_string())); + stream.get_module(Pubsub.Module.IDENTITY).publish.begin(stream, jid, NODE_DEVICELIST, id, node, null, true, () => { + try_make_node_public.begin(stream, NODE_DEVICELIST); + }); + } + publish_bundles_if_needed(stream, jid); + } + + foreach (StanzaNode device_node in node.get_subnodes("device")) { + device_list.add(device_node.get_attribute_int("id")); + } + device_list_loaded(jid, device_list); + + return device_list; + } + + public void fetch_bundles(XmppStream stream, Jid jid, Gee.List<int32> devices) { + Address address = new Address(jid.bare_jid.to_string(), 0); + foreach(int32 device_id in devices) { + if (!is_ignored_device(jid, device_id)) { + address.device_id = device_id; + try { + if (!store.contains_session(address)) { + fetch_bundle(stream, jid, device_id); + } + } catch (Error e) { + // Ignore + } + } + } + address.device_id = 0; // TODO: Hack to have address obj live longer + } + + public void fetch_bundle(XmppStream stream, Jid jid, int device_id, bool ignore_if_non_present = true) { + if (active_bundle_requests.add(jid.bare_jid.to_string() + @":$device_id")) { + debug("Asking for bundle for %s/%d", jid.bare_jid.to_string(), device_id); + stream.get_module(Pubsub.Module.IDENTITY).request(stream, jid.bare_jid, @"$NODE_BUNDLES:$device_id", (stream, jid, id, node) => { + on_other_bundle_result(stream, jid, device_id, id, node, ignore_if_non_present); + }); + } + } + + public void ignore_device(Jid jid, int32 device_id) { + if (device_id <= 0) return; + lock (device_ignore_time) { + device_ignore_time[jid.bare_jid.to_string() + @":$device_id"] = new DateTime.now_utc(); + } + } + + public void unignore_device(Jid jid, int32 device_id) { + if (device_id <= 0) return; + lock (device_ignore_time) { + device_ignore_time.unset(jid.bare_jid.to_string() + @":$device_id"); + } + } + + public bool is_ignored_device(Jid jid, int32 device_id) { + if (device_id <= 0) return true; + lock (device_ignore_time) { + string id = jid.bare_jid.to_string() + @":$device_id"; + if (device_ignore_time.has_key(id)) { + return new DateTime.now_utc().difference(device_ignore_time[id]) < IGNORE_TIME; + } + } + return false; + } + + public void clear_device_list(XmppStream stream) { + stream.get_module(Pubsub.Module.IDENTITY).delete_node(stream, null, NODE_DEVICELIST); + } + + private void on_other_bundle_result(XmppStream stream, Jid jid, int device_id, string? id, StanzaNode? node, bool ignore_if_non_present) { + if (node == null) { + // Device not registered, shouldn't exist + if (ignore_if_non_present) { + warning("Ignoring device %s/%d: No bundle", jid.bare_jid.to_string(), device_id); + stream.get_module(IDENTITY).ignore_device(jid, device_id); + } + bundle_fetch_failed(jid, device_id); + } else { + Bundle bundle = new Bundle(node); + if (bundle.is_valid) { + stream.get_module(IDENTITY).unignore_device(jid, device_id); + debug("Received bundle for %s/%d: %s", jid.bare_jid.to_string(), device_id, Base64.encode(bundle.identity_key.serialize())); + bundle_fetched(jid, device_id, bundle); + } else { + warning("Ignoring device %s/%d: Invalid bundle", jid.bare_jid.to_string(), device_id); + bundle_fetch_failed(jid, device_id); + } + } + stream.get_module(IDENTITY).active_bundle_requests.remove(jid.bare_jid.to_string() + @":$device_id"); + } + + public bool start_session(XmppStream stream, Jid jid, int32 device_id, Bundle bundle) { + bool fail = false; + int32 signed_pre_key_id = bundle.signed_pre_key_id; + ECPublicKey? signed_pre_key = bundle.signed_pre_key; + uint8[] signed_pre_key_signature = bundle.signed_pre_key_signature; + ECPublicKey? identity_key = bundle.identity_key; + + ArrayList<Bundle.PreKey> pre_keys = bundle.pre_keys; + if (signed_pre_key_id < 0 || signed_pre_key == null || identity_key == null || pre_keys.size == 0) { + fail = true; + } else { + int pre_key_idx = Random.int_range(0, pre_keys.size); + int32 pre_key_id = pre_keys[pre_key_idx].key_id; + ECPublicKey? pre_key = pre_keys[pre_key_idx].key; + if (pre_key_id < 0 || pre_key == null) { + fail = true; + } else { + Address address = new Address(jid.bare_jid.to_string(), device_id); + try { + if (store.contains_session(address)) { + return false; + } + debug("Starting new session for encryption with %s/%d", jid.bare_jid.to_string(), device_id); + SessionBuilder builder = store.create_session_builder(address); + builder.process_pre_key_bundle(create_pre_key_bundle(device_id, device_id, pre_key_id, pre_key, signed_pre_key_id, signed_pre_key, signed_pre_key_signature, identity_key)); + } catch (Error e) { + debug("Can't create session with %s/%d: %s", jid.bare_jid.to_string(), device_id, e.message); + fail = true; + } + address.device_id = 0; // TODO: Hack to have address obj live longer + } + } + if (fail) { + debug("Ignoring device %s/%d: Bad bundle: %s", jid.bare_jid.to_string(), device_id, bundle.node.to_string()); + stream.get_module(IDENTITY).ignore_device(jid, device_id); + } + return true; + } + + public void publish_bundles_if_needed(XmppStream stream, Jid jid) { + if (active_bundle_requests.add(jid.bare_jid.to_string() + @":$(store.local_registration_id)")) { + stream.get_module(Pubsub.Module.IDENTITY).request(stream, jid, @"$NODE_BUNDLES:$(store.local_registration_id)", on_self_bundle_result); + } + } + + private void on_self_bundle_result(XmppStream stream, Jid jid, string? id, StanzaNode? node) { + if (!Plugin.ensure_context()) return; + Map<int, ECPublicKey> keys = new HashMap<int, ECPublicKey>(); + ECPublicKey? identity_key = null; + int32 signed_pre_key_id = -1; + ECPublicKey? signed_pre_key = null; + SignedPreKeyRecord? signed_pre_key_record = null; + bool changed = false; + if (node == null) { + identity_key = store.identity_key_pair.public; + changed = true; + } else { + Bundle bundle = new Bundle(node); + foreach (Bundle.PreKey prekey in bundle.pre_keys) { + ECPublicKey? key = prekey.key; + if (key != null) { + keys[prekey.key_id] = (!)key; + } + } + identity_key = bundle.identity_key; + signed_pre_key_id = bundle.signed_pre_key_id; + signed_pre_key = bundle.signed_pre_key; + } + + try { + // Validate IdentityKey + if (identity_key == null || store.identity_key_pair.public.compare((!)identity_key) != 0) { + changed = true; + } + IdentityKeyPair identity_key_pair = store.identity_key_pair; + + // Validate signedPreKeyRecord + ID + if (signed_pre_key == null || signed_pre_key_id == -1 || !store.contains_signed_pre_key(signed_pre_key_id) || store.load_signed_pre_key(signed_pre_key_id).key_pair.public.compare((!)signed_pre_key) != 0) { + signed_pre_key_id = Random.int_range(1, int32.MAX); // TODO: No random, use ordered number + signed_pre_key_record = Plugin.get_context().generate_signed_pre_key(identity_key_pair, signed_pre_key_id); + store.store_signed_pre_key((!)signed_pre_key_record); + changed = true; + } else { + signed_pre_key_record = store.load_signed_pre_key(signed_pre_key_id); + } + + // Validate PreKeys + Set<PreKeyRecord> pre_key_records = new HashSet<PreKeyRecord>(); + foreach (var entry in keys.entries) { + if (store.contains_pre_key(entry.key)) { + PreKeyRecord record = store.load_pre_key(entry.key); + if (record.key_pair.public.compare(entry.value) == 0) { + pre_key_records.add(record); + } + } + } + int new_keys = NUM_KEYS_TO_PUBLISH - pre_key_records.size; + if (new_keys > 0) { + int32 next_id = Random.int_range(1, int32.MAX); // TODO: No random, use ordered number + Set<PreKeyRecord> new_records = Plugin.get_context().generate_pre_keys((uint)next_id, (uint)new_keys); + pre_key_records.add_all(new_records); + foreach (PreKeyRecord record in new_records) { + store.store_pre_key(record); + } + changed = true; + } + + if (changed) { + publish_bundles.begin(stream, (!)signed_pre_key_record, identity_key_pair, pre_key_records, (int32) store.local_registration_id); + } + } catch (Error e) { + warning(@"Unexpected error while publishing bundle: $(e.message)\n"); + } + stream.get_module(IDENTITY).active_bundle_requests.remove(jid.bare_jid.to_string() + @":$(store.local_registration_id)"); + } + + public async void publish_bundles(XmppStream stream, SignedPreKeyRecord signed_pre_key_record, IdentityKeyPair identity_key_pair, Set<PreKeyRecord> pre_key_records, int32 device_id) throws Error { + ECKeyPair tmp; + StanzaNode bundle = new StanzaNode.build("bundle", NS_URI) + .add_self_xmlns() + .put_node(new StanzaNode.build("signedPreKeyPublic", NS_URI) + .put_attribute("signedPreKeyId", signed_pre_key_record.id.to_string()) + .put_node(new StanzaNode.text(Base64.encode((tmp = signed_pre_key_record.key_pair).public.serialize())))) + .put_node(new StanzaNode.build("signedPreKeySignature", NS_URI) + .put_node(new StanzaNode.text(Base64.encode(signed_pre_key_record.signature)))) + .put_node(new StanzaNode.build("identityKey", NS_URI) + .put_node(new StanzaNode.text(Base64.encode(identity_key_pair.public.serialize())))); + StanzaNode prekeys = new StanzaNode.build("prekeys", NS_URI); + foreach (PreKeyRecord pre_key_record in pre_key_records) { + prekeys.put_node(new StanzaNode.build("preKeyPublic", NS_URI) + .put_attribute("preKeyId", pre_key_record.id.to_string()) + .put_node(new StanzaNode.text(Base64.encode(pre_key_record.key_pair.public.serialize())))); + } + bundle.put_node(prekeys); + + string node_id = @"$NODE_BUNDLES:$device_id"; + yield stream.get_module(Pubsub.Module.IDENTITY).publish(stream, null, node_id, "1", bundle); + yield try_make_node_public(stream, node_id); + + } + + private async void try_make_node_public(XmppStream stream, string node_id) { + DataForms.DataForm? data_form = yield stream.get_module(Pubsub.Module.IDENTITY).request_node_config(stream, null, node_id); + if (data_form == null) return; + + foreach (DataForms.DataForm.Field field in data_form.fields) { + if (field.var == "pubsub#access_model" && field.get_value_string() != Pubsub.ACCESS_MODEL_OPEN) { + field.set_value_string(Pubsub.ACCESS_MODEL_OPEN); + yield stream.get_module(Pubsub.Module.IDENTITY).submit_node_config(stream, data_form, node_id); + break; + } + } + } + + public override string get_ns() { + return NS_URI; + } + + public override string get_id() { + return IDENTITY.id; + } +} + +} diff --git a/main/src/plugins/omemo/trust_level.vala b/main/src/plugins/omemo/trust_level.vala new file mode 100644 index 00000000..afcfb0fb --- /dev/null +++ b/main/src/plugins/omemo/trust_level.vala @@ -0,0 +1,15 @@ +namespace Dino.Plugins.Omemo { + +public enum TrustLevel { + VERIFIED, + TRUSTED, + UNTRUSTED, + UNKNOWN; + + public string to_string() { + int val = this; + return val.to_string(); + } +} + +} diff --git a/main/src/plugins/omemo/ui/bad_messages_populator.vala b/main/src/plugins/omemo/ui/bad_messages_populator.vala new file mode 100644 index 00000000..c291a422 --- /dev/null +++ b/main/src/plugins/omemo/ui/bad_messages_populator.vala @@ -0,0 +1,198 @@ +using Gee; +using Gtk; + +using Dino.Entities; +using Dino.Qlite; +using Xmpp; + +namespace Dino.Plugins.Omemo { + +public enum BadnessType { + UNTRUSTED, + UNDECRYPTABLE +} + +public class BadMessagesPopulator : Plugins.ConversationItemPopulator, Plugins.ConversationAdditionPopulator, Object { + + public string id { get { return "bad_omemo_messages"; } } + + private StreamInteractor stream_interactor; + private Plugin plugin; + private Database db; + + private Conversation? current_conversation; + private Plugins.ConversationItemCollection? item_collection; + + private Gee.List<BadMessageItem> bad_items = new ArrayList<BadMessageItem>(); + + public BadMessagesPopulator(StreamInteractor stream_interactor, Plugin plugin) { + this.stream_interactor = stream_interactor; + this.plugin = plugin; + this.db = plugin.db; + + plugin.trust_manager.bad_message_state_updated.connect((account, jid, device_id) => { + clear_state(); + init_state(); + }); + } + + private void init_state() { + if (current_conversation == null) return; + if (current_conversation.type_ == Conversation.Type.GROUPCHAT_PM) return; + + var qry = db.identity_meta.select() + .join_with(db.identity, db.identity.id, db.identity_meta.identity_id) + .with(db.identity.account_id, "=", current_conversation.account.id) + .where("last_message_untrusted is not NULL OR last_message_undecryptable is not NULL"); + + switch (current_conversation.type_) { + case Conversation.Type.CHAT: + qry.with(db.identity_meta.address_name, "=", current_conversation.counterpart.to_string()); + break; + case Conversation.Type.GROUPCHAT: + bool is_private = stream_interactor.get_module(MucManager.IDENTITY).is_private_room(current_conversation.account, current_conversation.counterpart); + if (!is_private) return; + + var list = stream_interactor.get_module(MucManager.IDENTITY).get_offline_members(current_conversation.counterpart, current_conversation.account); + if (list == null || list.is_empty) return; + + var selection = new StringBuilder(); + string[] selection_args = {}; + foreach (Jid jid in list) { + if (selection.len == 0) { + selection.append(@" ($(db.identity_meta.address_name) = ?"); + } else { + selection.append(@" OR $(db.identity_meta.address_name) = ?"); + } + selection_args += jid.to_string(); + } + selection.append(")"); + qry.where(selection.str, selection_args); + break; + case Conversation.Type.GROUPCHAT_PM: + break; + } + + foreach (Row row in qry) { + Jid jid = Jid.from_string(row[db.identity_meta.address_name]); + if (!db.identity_meta.last_message_untrusted.is_null(row)) { + DateTime time = new DateTime.from_unix_utc(row[db.identity_meta.last_message_untrusted]); + var item = new BadMessageItem(plugin, current_conversation, jid, time, BadnessType.UNTRUSTED); + bad_items.add(item); + item_collection.insert_item(item); + } + if (!db.identity_meta.last_message_undecryptable.is_null(row)) { + DateTime time = new DateTime.from_unix_utc(row[db.identity_meta.last_message_undecryptable]); + var item = new BadMessageItem(plugin, current_conversation, jid, time, BadnessType.UNDECRYPTABLE); + bad_items.add(item); + item_collection.insert_item(item); + } + } + } + + private void clear_state() { + foreach (BadMessageItem bad_item in bad_items) { + item_collection.remove_item(bad_item); + } + bad_items.clear(); + } + + public void init(Conversation conversation, Plugins.ConversationItemCollection item_collection, Plugins.WidgetType type) { + current_conversation = conversation; + this.item_collection = item_collection; + + init_state(); + } + + public void close(Conversation conversation) { + clear_state(); + } + + public void populate_timespan(Conversation conversation, DateTime after, DateTime before) { } +} + +public class BadMessageItem : Plugins.MetaConversationItem { + + private Plugin plugin; + private Conversation conversation; + private Jid problem_jid; + private BadnessType badness_type; + + public BadMessageItem(Plugin plugin, Conversation conversation, Jid jid, DateTime date, BadnessType badness_type) { + this.plugin = plugin; + this.conversation = conversation; + this.problem_jid = jid; + this.time = date; + this.badness_type = badness_type; + } + + public override Object? get_widget(Plugins.ConversationItemWidgetInterface outer, Plugins.WidgetType widget_type) { + return new BadMessagesWidget(plugin, conversation, problem_jid, badness_type); + } + + public override Gee.List<Plugins.MessageAction>? get_item_actions(Plugins.WidgetType type) { return null; } +} + +public class BadMessagesWidget : Box { + private Plugin plugin; + private Conversation conversation; + private Jid jid; + private Label label; + + public BadMessagesWidget(Plugin plugin, Conversation conversation, Jid jid, BadnessType badness_type) { + Object(orientation:Orientation.HORIZONTAL, spacing:5); + + this.plugin = plugin; + this.conversation = conversation; + this.jid = jid; + this.halign = Align.CENTER; + this.visible = true; + + string who = ""; + if (conversation.type_ == Conversation.Type.CHAT) { + who = Dino.get_participant_display_name(plugin.app.stream_interactor, conversation, jid); + } else if (conversation.type_ == Conversation.Type.GROUPCHAT) { + who = jid.to_string(); + // `jid` is a real JID. In MUCs, try to show nicks instead (given that the JID is currently online) + var occupants = plugin.app.stream_interactor.get_module(MucManager.IDENTITY).get_occupants(conversation.counterpart, conversation.account); + if (occupants == null) return; + foreach (Jid occupant in occupants) { + if (jid.equals_bare(plugin.app.stream_interactor.get_module(MucManager.IDENTITY).get_real_jid(occupant, conversation.account))) { + who = occupant.resourcepart; + } + } + } + + string warning_text = ""; + if (badness_type == BadnessType.UNTRUSTED) { + warning_text = _("%s has been using an untrusted device. You won't see messages from devices that you do not trust.").printf(who) + + " <a href=\"\">%s</a>".printf(_("Manage devices")); + } else { + warning_text += _("%s does not trust this device. That means, you might be missing messages.").printf(who); + } + label = new Label(warning_text) { margin_start=70, margin_end=70, justify=Justification.CENTER, use_markup=true, selectable=true, wrap=true, wrap_mode=Pango.WrapMode.WORD_CHAR, hexpand=true }; + label.add_css_class("dim-label"); + this.append(label); + + if (badness_type == BadnessType.UNTRUSTED) { + label.activate_link.connect(on_label_activate_link); + } + } + + private bool on_label_activate_link() { + var variant = new Variant.tuple(new Variant[] {new Variant.int32(conversation.id), new Variant.string("encryption")}); + GLib.Application.get_default().activate_action("open-conversation-details", variant); + return false; + } + + public override void dispose() { + if (label != null) { + label.unparent(); + label.dispose(); + label = null; + } + base.dispose(); + } +} + +} diff --git a/main/src/plugins/omemo/ui/call_encryption_entry.vala b/main/src/plugins/omemo/ui/call_encryption_entry.vala new file mode 100644 index 00000000..04f0443c --- /dev/null +++ b/main/src/plugins/omemo/ui/call_encryption_entry.vala @@ -0,0 +1,58 @@ +using Gtk; + +using Dino.Entities; +using Dino.Qlite; +using Xmpp; + +namespace Dino.Plugins.Omemo { + + public class CallEncryptionEntry : Plugins.CallEncryptionEntry, Object { + private Database db; + + public CallEncryptionEntry(Database db) { + this.db = db; + } + + public Plugins.CallEncryptionWidget? get_widget(Account account, Xmpp.Xep.Jingle.ContentEncryption encryption) { + DtlsSrtpVerificationDraft.OmemoContentEncryption? omemo_encryption = encryption as DtlsSrtpVerificationDraft.OmemoContentEncryption; + if (omemo_encryption == null) return null; + + int identity_id = db.identity.get_id(account.id); + Row? device = db.identity_meta.get_device(identity_id, omemo_encryption.jid.to_string(), omemo_encryption.sid); + if (device == null) return null; + TrustLevel trust = (TrustLevel) device[db.identity_meta.trust_level]; + + return new CallEncryptionWidget(trust); + } + } + + public class CallEncryptionWidget : Plugins.CallEncryptionWidget, Object { + + string? title = null; + string? icon = null; + bool should_show_keys = false; + + public CallEncryptionWidget(TrustLevel trust) { + if (trust == TrustLevel.VERIFIED) { + title = "This call is <b>encrypted and verified</b> with OMEMO."; + icon = "dino-security-high-symbolic"; + should_show_keys = false; + } else { + title = "This call is encrypted with OMEMO."; + should_show_keys = true; + } + } + + public string? get_title() { + return title; + } + + public string? get_icon_name() { + return icon; + } + + public bool show_keys() { + return should_show_keys; + } + } +} diff --git a/main/src/plugins/omemo/ui/contact_details_provider.vala b/main/src/plugins/omemo/ui/contact_details_provider.vala new file mode 100644 index 00000000..1b61f31d --- /dev/null +++ b/main/src/plugins/omemo/ui/contact_details_provider.vala @@ -0,0 +1,28 @@ +using Gtk; +using Gee; +using Dino.Entities; + +namespace Dino.Plugins.Omemo { + +public class ContactDetailsProvider : Plugins.ContactDetailsProvider, Object { + public string id { get { return "omemo_info"; } } + public string tab { get { return "encryption"; } } + + private Plugin plugin; + + public ContactDetailsProvider(Plugin plugin) { + this.plugin = plugin; + } + + public void populate(Conversation conversation, Plugins.ContactDetails contact_details, WidgetType type) { } + + public Object? get_widget(Conversation conversation) { + if (conversation.type_ != Conversation.Type.CHAT) return null; + + var widget = new OmemoPreferencesWidget(plugin); + widget.set_jid(conversation.account, conversation.counterpart); + return widget; + } +} + +} diff --git a/main/src/plugins/omemo/ui/device_notification_populator.vala b/main/src/plugins/omemo/ui/device_notification_populator.vala new file mode 100644 index 00000000..7829d89c --- /dev/null +++ b/main/src/plugins/omemo/ui/device_notification_populator.vala @@ -0,0 +1,90 @@ +using Dino.Entities; +using Xmpp; +using Gtk; + +namespace Dino.Plugins.Omemo { + +public class DeviceNotificationPopulator : NotificationPopulator, Object { + + public string id { get { return "device_notification"; } } + + private StreamInteractor? stream_interactor; + private Plugin plugin; + private Conversation? current_conversation; + private NotificationCollection? notification_collection; + private ConversationNotification notification; + + public DeviceNotificationPopulator(Plugin plugin, StreamInteractor stream_interactor) { + this.stream_interactor = stream_interactor; + this.plugin = plugin; + + stream_interactor.account_added.connect(on_account_added); + } + + public void init(Conversation conversation, NotificationCollection notification_collection, Plugins.WidgetType type) { + current_conversation = conversation; + this.notification_collection = notification_collection; + if (plugin.has_new_devices(conversation.account, conversation.counterpart) && conversation.type_ == Conversation.Type.CHAT) { + display_notification(); + } + } + + public void close(Conversation conversation) { + notification = null; + } + + private void display_notification() { + if (notification == null) { + notification = new ConversationNotification(plugin, current_conversation); + notification.should_hide.connect(should_hide); + notification_collection.add_meta_notification(notification); + } + } + + public void should_hide() { + if (!plugin.has_new_devices(current_conversation.account, current_conversation.counterpart) && notification != null){ + notification_collection.remove_meta_notification(notification); + notification = null; + } + } + + private void on_account_added(Account account) { + stream_interactor.module_manager.get_module(account, StreamModule.IDENTITY).bundle_fetched.connect_after((jid, device_id, bundle) => { + if (current_conversation != null && jid.equals(current_conversation.counterpart) && plugin.has_new_devices(current_conversation.account, current_conversation.counterpart)) { + display_notification(); + } + }); + } +} + +private class ConversationNotification : MetaConversationNotification { + private Widget widget; + private Plugin plugin; + private Jid jid; + private Account account; + public signal void should_hide(); + + public ConversationNotification(Plugin plugin, Conversation conversation) { + this.plugin = plugin; + this.jid = jid; + this.account = account; + + Box box = new Box(Orientation.HORIZONTAL, 5); + Button manage_button = new Button.with_label(_("Manage")); + manage_button.clicked.connect(() => { + manage_button.activate(); + + var variant = new Variant.tuple(new Variant[] {new Variant.int32(conversation.id), new Variant.string("encryption")}); + GLib.Application.get_default().activate_action("open-conversation-details", variant); + }); + box.append(new Label(_("This contact has new devices")) { margin_end=10 }); + box.append(manage_button); + widget = box; + } + + public override Object? get_widget(WidgetType type) { + return widget; + } +} + +} diff --git a/main/src/plugins/omemo/ui/encryption_list_entry.vala b/main/src/plugins/omemo/ui/encryption_list_entry.vala new file mode 100644 index 00000000..dfec813a --- /dev/null +++ b/main/src/plugins/omemo/ui/encryption_list_entry.vala @@ -0,0 +1,78 @@ +using Gtk; + +using Dino.Entities; +using Dino.Qlite; +using Xmpp; + +namespace Dino.Plugins.Omemo { + +public class EncryptionListEntry : Plugins.EncryptionListEntry, Object { + private Plugin plugin; + private Database db; + + public EncryptionListEntry(Plugin plugin) { + this.plugin = plugin; + this.db = plugin.db; + } + + public Entities.Encryption encryption { get { + return Entities.Encryption.OMEMO; + }} + + public string name { get { + return "OMEMO"; + }} + + public Object? get_encryption_icon(Entities.Conversation conversation, ContentItem content_item) { + return null; + } + + public string? get_encryption_icon_name(Entities.Conversation conversation, ContentItem content_item) { + if (content_item.encryption != encryption) return null; + + RowOption row = db.content_item_meta.select( { db.identity_meta.trust_level } ).with(db.content_item_meta.content_item_id, "=", content_item.id) + .join_on(db.identity_meta, @"$(db.identity_meta.address_name)=$(db.content_item_meta.address_name) AND $(db.identity_meta.device_id)=$(db.content_item_meta.device_id)") + .single().row(); + + + if (row.is_present() && (TrustLevel) row[db.identity_meta.trust_level] == TrustLevel.VERIFIED) { + return "dino-security-high-symbolic"; + } + return null; + } + + public void encryption_activated(Entities.Conversation conversation, Plugins.SetInputFieldStatus input_status_callback) { + encryption_activated_async.begin(conversation, input_status_callback); + } + + public async void encryption_activated_async(Entities.Conversation conversation, Plugins.SetInputFieldStatus input_status_callback) { + if (conversation.type_ == Conversation.Type.GROUPCHAT_PM) { + input_status_callback(new Plugins.InputFieldStatus("Can't use encryption in a groupchat private message.", Plugins.InputFieldStatus.MessageType.ERROR, Plugins.InputFieldStatus.InputState.NO_SEND)); + return; + } + MucManager muc_manager = plugin.app.stream_interactor.get_module(MucManager.IDENTITY); + Manager omemo_manager = plugin.app.stream_interactor.get_module(Manager.IDENTITY); + + if (muc_manager.is_private_room(conversation.account, conversation.counterpart)) { + var offline_members = muc_manager.get_offline_members(conversation.counterpart, conversation.account); + if (offline_members == null) { + // We don't store offline members yet, and it'll be null if we're offline + return; + } + foreach (Jid offline_member in offline_members) { + bool ok = yield omemo_manager.ensure_get_keys_for_jid(conversation.account, offline_member); + if (!ok) { + input_status_callback(new Plugins.InputFieldStatus("A member does not support OMEMO: %s".printf(offline_member.to_string()), Plugins.InputFieldStatus.MessageType.ERROR, Plugins.InputFieldStatus.InputState.NO_SEND)); + return; + } + } + return; + } + + if (!(yield omemo_manager.ensure_get_keys_for_jid(conversation.account, conversation.counterpart.bare_jid))) { + input_status_callback(new Plugins.InputFieldStatus("This contact does not support %s encryption".printf("OMEMO"), Plugins.InputFieldStatus.MessageType.ERROR, Plugins.InputFieldStatus.InputState.NO_SEND)); + } + } + +} +} diff --git a/main/src/plugins/omemo/ui/encryption_preferences_entry.vala b/main/src/plugins/omemo/ui/encryption_preferences_entry.vala new file mode 100644 index 00000000..7f53a1b2 --- /dev/null +++ b/main/src/plugins/omemo/ui/encryption_preferences_entry.vala @@ -0,0 +1,324 @@ +using Gee; +using Gtk; +using Qrencode; + +using Dino.Crypto.Omemo; +using Dino.Entities; +using Dino.Qlite; +using Omemo; +using Xmpp; + +namespace Dino.Plugins.Omemo { + +public class OmemoPreferencesEntry : Plugins.EncryptionPreferencesEntry { + + OmemoPreferencesWidget widget; + Plugin plugin; + + public OmemoPreferencesEntry(Plugin plugin) { + this.plugin = plugin; + } + + public override Object? get_widget(Account account, WidgetType type) { + if (type != WidgetType.GTK4) return null; + var widget = new OmemoPreferencesWidget(plugin); + widget.set_jid(account, account.bare_jid); + return widget; + } + + public override string id { get { return "omemo_preferences_entryption"; }} +} + +[GtkTemplate (ui = "/im/dino/Dino/encryption_preferences_entry.ui")] +public class OmemoPreferencesWidget : Adw.PreferencesGroup { + private Plugin plugin; + private Account account; + private Jid jid; + private int identity_id = 0; + private Store store; + private Set<uint32> displayed_ids = new HashSet<uint32>(); + + [GtkChild] private unowned Adw.SwitchRow automatically_accept_new_row; + [GtkChild] private unowned Adw.SwitchRow encrypt_by_default_row; + [GtkChild] private unowned Label new_keys_label; + + [GtkChild] private unowned Adw.PreferencesGroup keys_preferences_group; + [GtkChild] private unowned ListBox new_keys_listbox; + [GtkChild] private unowned Picture qrcode_picture; + [GtkChild] private unowned Popover qrcode_popover; + + private ArrayList<Widget> keys_preferences_group_children = new ArrayList<Widget>(); + + construct { + // If we set the strings in the .ui file, they don't get translated + encrypt_by_default_row.title = _("OMEMO by default"); + encrypt_by_default_row.subtitle = _("Enable OMEMO encryption for new conversations"); + automatically_accept_new_row.title = _("Encrypt to new devices"); + automatically_accept_new_row.subtitle = _("Automatically encrypt to new devices from this contact."); + new_keys_label.label = _("New keys"); + } + + public OmemoPreferencesWidget(Plugin plugin) { + this.plugin = plugin; + } + + public void set_jid(Account account, Jid jid) { + this.account = account; + this.jid = jid; + this.identity_id = plugin.db.identity.get_id(account.id); + if (identity_id <= 0) { + warning("OmemoPreferencesWidget missing identity_id"); + return; + } + + automatically_accept_new_row.active = plugin.db.trust.get_blind_trust(identity_id, jid.bare_jid.to_string(), true); + automatically_accept_new_row.notify["active"].connect((obj, _) => { on_auto_accept_toggled(((Adw.SwitchRow) obj).active); }); + + encrypt_by_default_row.visible = account.bare_jid.equals_bare(jid); + encrypt_by_default_row.active = plugin.app.settings.get_default_encryption(account) != Encryption.NONE; + encrypt_by_default_row.notify["active"].connect((obj, _) => { on_omemo_by_default_toggled(((Adw.SwitchRow) obj).active); }); + + Dino.Application? app = Application.get_default() as Dino.Application; + if (app != null) { + store = app.stream_interactor.module_manager.get_module(account, StreamModule.IDENTITY).store; + } + + redraw_key_list(); + + // Check for unknown devices + fetch_unknown_bundles(); + } + + private void redraw_key_list() { + // Remove current widgets + foreach (var widget in keys_preferences_group_children) { + keys_preferences_group.remove(widget); + } + keys_preferences_group_children.clear(); + + // Dialog opened from the account settings menu + // Show the fingerprint for this device separately with buttons for a qrcode and to copy + if(jid.equals(account.bare_jid)) { + automatically_accept_new_row.subtitle = _("New encryption keys from your other devices will be accepted automatically."); + add_own_fingerprint(); + } + + //Show the normal devicelist + var own_id = plugin.db.identity.row_with(plugin.db.identity.account_id, account.id)[plugin.db.identity.device_id]; + foreach (Row device in plugin.db.identity_meta.get_known_devices(identity_id, jid.to_string())) { + if(jid.equals(account.bare_jid) && device[plugin.db.identity_meta.device_id] == own_id) { + // If this is our own account, don't show this device twice (did it separately already) + continue; + } + add_fingerprint(device, (TrustLevel) device[plugin.db.identity_meta.trust_level]); + } + + //Show any new devices for which the user must decide whether to accept or reject + foreach (Row device in plugin.db.identity_meta.get_new_devices(identity_id, jid.to_string())) { + add_new_fingerprint(device); + } + } + + private static string escape_for_iri_path_segment(string s) { + // from RFC 3986, 2.2. Reserved Characters: + string SUB_DELIMS = "!$&'()*+,;="; + // from RFC 3986, 3.3. Path (pchar without unreserved and pct-encoded): + string ALLOWED_RESERVED_CHARS = SUB_DELIMS + ":@"; + return GLib.Uri.escape_string(s, ALLOWED_RESERVED_CHARS, true); + } + + private void fetch_unknown_bundles() { + Dino.Application app = Application.get_default() as Dino.Application; + XmppStream? stream = app.stream_interactor.get_stream(account); + if (stream == null) return; + StreamModule? module = stream.get_module(StreamModule.IDENTITY); + if (module == null) return; + module.bundle_fetched.connect_after((bundle_jid, device_id, bundle) => { + if (bundle_jid.equals(jid) && !displayed_ids.contains(device_id)) { + redraw_key_list(); + } + }); + foreach (Row device in plugin.db.identity_meta.get_unknown_devices(identity_id, jid.to_string())) { + try { + module.fetch_bundle(stream, Jid.from_string(device[plugin.db.identity_meta.address_name]), device[plugin.db.identity_meta.device_id], false); + } catch (InvalidJidError e) { + warning("Ignoring device with invalid Jid: %s", e.message); + } + } + } + + private void add_own_fingerprint() { + string own_b64 = plugin.db.identity.row_with(plugin.db.identity.account_id, account.id)[plugin.db.identity.identity_key_public_base64]; + string fingerprint = fingerprint_from_base64(own_b64); + + var own_action_box = new Box(Orientation.HORIZONTAL, 6); + var show_qrcode_button = new MenuButton() { icon_name="dino-qr-code-symbolic", valign=Align.CENTER }; + own_action_box.append(show_qrcode_button); + var copy_button = new Button() { icon_name="dino-edit-copy-symbolic", valign=Align.CENTER }; + copy_button.clicked.connect(() => { copy_button.get_clipboard().set_text(fingerprint); }); + own_action_box.append(copy_button); + + Adw.ActionRow action_row = new Adw.ActionRow() { use_markup = true }; + action_row.title = _("This device"); + action_row.subtitle = fingerprint_markup(fingerprint_from_base64(own_b64)); + action_row.add_suffix(own_action_box); + add_key_row(action_row); + + // Create and set QR code popover + int sid = plugin.db.identity.row_with(plugin.db.identity.account_id, account.id)[plugin.db.identity.device_id]; + var iri_query = @"omemo-sid-$(sid)=$(fingerprint)"; + string iri = GLib.Uri.join(UriFlags.NONE, "xmpp", null, null, 0, jid.to_string(), iri_query, null); + + const int QUIET_ZONE_MODULES = 4; // MUST be at least 4 + const int MODULE_SIZE_PX = 4; // arbitrary + var qr_paintable = new QRcode(iri, 2) + .to_paintable(MODULE_SIZE_PX * qrcode_picture.scale_factor); + qrcode_picture.paintable = qr_paintable; + qrcode_picture.margin_top = qrcode_picture.margin_end = + qrcode_picture.margin_bottom = qrcode_picture.margin_start = QUIET_ZONE_MODULES * MODULE_SIZE_PX; + qrcode_popover.add_css_class("qrcode-container"); + + show_qrcode_button.popover = qrcode_popover; + } + + private void add_fingerprint(Row device, TrustLevel trust) { + string key_base64 = device[plugin.db.identity_meta.identity_key_public_base64]; + bool key_active = device[plugin.db.identity_meta.now_active]; + if (store != null) { + try { + Address address = new Address(jid.to_string(), device[plugin.db.identity_meta.device_id]); + SessionRecord? session = null; + if (store.contains_session(address)) { + session = store.load_session(address); + string session_key_base64 = Base64.encode(session.state.remote_identity_key.serialize()); + if (key_base64 != session_key_base64) { + critical("Session and database identity key mismatch!"); + key_base64 = session_key_base64; + } + } + } catch (Error e) { + print("Error while reading session store: %s", e.message); + } + } + + if (device[plugin.db.identity_meta.now_active]) { + Adw.ActionRow action_row = new Adw.ActionRow() { use_markup = true }; + action_row.activated.connect(() => { + Row updated_device = plugin.db.identity_meta.get_device(device[plugin.db.identity_meta.identity_id], device[plugin.db.identity_meta.address_name], device[plugin.db.identity_meta.device_id]); + ManageKeyDialog manage_dialog = new ManageKeyDialog(updated_device, plugin.db); + manage_dialog.set_transient_for((Gtk.Window) get_root()); + manage_dialog.present(); + manage_dialog.response.connect((response) => { + update_stored_trust(response, updated_device); + redraw_key_list(); + }); + }); + action_row.activatable = true; + action_row.title = account.bare_jid.equals_bare(jid) ? _("Other device") : _("Device"); + action_row.subtitle = fingerprint_markup(fingerprint_from_base64(key_base64)); + string trust_str = _("Accepted"); + switch(trust) { + case TrustLevel.UNTRUSTED: + trust_str = _("Rejected"); + break; + case TrustLevel.VERIFIED: + trust_str = _("Verified"); + break; + case TrustLevel.TRUSTED: + case TrustLevel.UNKNOWN: + default: + break; + } + + action_row.add_suffix(new Label(trust_str)); + add_key_row(action_row); + } + displayed_ids.add(device[plugin.db.identity_meta.device_id]); + } + + private bool on_auto_accept_toggled(bool active) { + plugin.trust_manager.set_blind_trust(account, jid, active); + + if (active) { + int identity_id = plugin.db.identity.get_id(account.id); + if (identity_id < 0) return false; + + foreach (Row device in plugin.db.identity_meta.get_new_devices(identity_id, jid.to_string())) { + plugin.trust_manager.set_device_trust(account, jid, device[plugin.db.identity_meta.device_id], TrustLevel.TRUSTED); + add_fingerprint(device, TrustLevel.TRUSTED); + } + } + return false; + } + + private bool on_omemo_by_default_toggled(bool active) { + var encryption_value = active ? Encryption.OMEMO : Encryption.NONE; + plugin.app.settings.set_default_encryption(account, encryption_value); + return false; + } + + private void update_stored_trust(int response, Row device) { + switch (response) { + case TrustLevel.TRUSTED: + plugin.trust_manager.set_device_trust(account, jid, device[plugin.db.identity_meta.device_id], TrustLevel.TRUSTED); + break; + case TrustLevel.UNTRUSTED: + plugin.trust_manager.set_device_trust(account, jid, device[plugin.db.identity_meta.device_id], TrustLevel.UNTRUSTED); + break; + case TrustLevel.VERIFIED: + plugin.trust_manager.set_device_trust(account, jid, device[plugin.db.identity_meta.device_id], TrustLevel.VERIFIED); + plugin.trust_manager.set_blind_trust(account, jid, false); + automatically_accept_new_row.active = false; + break; + } + } + + private void add_new_fingerprint(Row device) { + Adw.ActionRow action_row = new Adw.ActionRow() { use_markup = true }; + action_row.title = _("New device"); + action_row.subtitle = fingerprint_markup(fingerprint_from_base64(device[plugin.db.identity_meta.identity_key_public_base64])); + + Button accept_button = new Button() { visible = true, valign = Align.CENTER, hexpand = true }; + accept_button.set_icon_name("dino-check-plain-symbolic"); // using .image = sets .image-button. Together with .suggested/destructive action that breaks the button Adwaita + accept_button.add_css_class("suggested-action"); + accept_button.tooltip_text = _("Accept key"); + + Button reject_button = new Button() { visible = true, valign = Align.CENTER, hexpand = true }; + reject_button.set_icon_name("action-unavailable-symbolic"); + reject_button.add_css_class("destructive-action"); + reject_button.tooltip_text = _("Reject key"); + + accept_button.clicked.connect(() => { + plugin.trust_manager.set_device_trust(account, jid, device[plugin.db.identity_meta.device_id], TrustLevel.TRUSTED); + add_fingerprint(device, TrustLevel.TRUSTED); + remove_key_row(action_row); + }); + + reject_button.clicked.connect(() => { + plugin.trust_manager.set_device_trust(account, jid, device[plugin.db.identity_meta.device_id], TrustLevel.UNTRUSTED); + add_fingerprint(device, TrustLevel.UNTRUSTED); + remove_key_row(action_row); + }); + + Box control_box = new Box(Gtk.Orientation.HORIZONTAL, 0) { visible = true, hexpand = true }; + control_box.append(accept_button); + control_box.append(reject_button); + control_box.add_css_class("linked"); // .linked: Visually link the accept / reject buttons + + action_row.add_suffix(control_box); + + add_key_row(action_row); + displayed_ids.add(device[plugin.db.identity_meta.device_id]); + } + + private void add_key_row(Adw.PreferencesRow widget) { + keys_preferences_group.add(widget); + keys_preferences_group_children.add(widget); + } + + private void remove_key_row(Adw.PreferencesRow widget) { + keys_preferences_group.remove(widget); + keys_preferences_group_children.remove(widget); + } +} +}
\ No newline at end of file diff --git a/main/src/plugins/omemo/ui/manage_key_dialog.vala b/main/src/plugins/omemo/ui/manage_key_dialog.vala new file mode 100644 index 00000000..59e6db24 --- /dev/null +++ b/main/src/plugins/omemo/ui/manage_key_dialog.vala @@ -0,0 +1,181 @@ +using Gtk; + +using Dino.Qlite; + +namespace Dino.Plugins.Omemo { + +[GtkTemplate (ui = "/im/dino/Dino/manage_key_dialog.ui")] +public class ManageKeyDialog : Gtk.Dialog { + + [GtkChild] private unowned Stack manage_stack; + + [GtkChild] private unowned Button cancel_button; + [GtkChild] private unowned Button ok_button; + + [GtkChild] private unowned Label main_desc_label; + [GtkChild] private unowned ListBox main_action_list; + + [GtkChild] private unowned Image confirm_image; + [GtkChild] private unowned Label confirm_title_label; + [GtkChild] private unowned Label confirm_desc_label; + + [GtkChild] private unowned Label verify_label; + [GtkChild] private unowned Label compare_fingerprint_label; + [GtkChild] private unowned Button verify_yes_button; + [GtkChild] private unowned Button verify_no_button; + + private Row device; + private Database db; + + private bool return_to_main; + private int current_response; + + construct { + // If we set the strings in the .ui file, they don't get translated + this.title = _("Manage Key"); + compare_fingerprint_label.label = _("Compare the fingerprint, character by character, with the one shown on your contact's device."); + verify_no_button.label = _("Fingerprints differ"); + verify_yes_button.label = _("Fingerprints match"); + cancel_button.label = _("Cancel"); + ok_button.label = _("Confirm"); + } + + public ManageKeyDialog(Row device, Database db) { + Object(use_header_bar : 1); + + this.device = device; + this.db = db; + + setup_main_screen(); + setup_verify_screen(); + + cancel_button.clicked.connect(handle_cancel); + ok_button.clicked.connect(() => { + response(current_response); + close(); + }); + + verify_yes_button.clicked.connect(() => { + confirm_image.set_from_icon_name("security-high-symbolic"); + confirm_title_label.label = _("Verify key"); + confirm_desc_label.set_markup(_("Future messages sent by %s from the device that uses this key will be highlighted accordingly in the chat window.").printf(@"<b>$(device[db.identity_meta.address_name])</b>")); + manage_stack.set_visible_child_name("confirm"); + ok_button.sensitive = true; + return_to_main = false; + current_response = TrustLevel.VERIFIED; + }); + + verify_no_button.clicked.connect(() => { + return_to_main = false; + confirm_image.set_from_icon_name("dino-dialog-warning-symbolic"); + confirm_title_label.label = _("Fingerprints do not match"); + confirm_desc_label.set_markup(_("Please verify that you are comparing the correct fingerprint. If fingerprints do not match, %s's account may be compromised and you should consider rejecting this key.").printf(@"<b>$(device[db.identity_meta.address_name])</b>")); + manage_stack.set_visible_child_name("confirm"); + }); + } + + private void handle_cancel() { + if (manage_stack.get_visible_child_name() == "main") close(); + + if (manage_stack.get_visible_child_name() == "verify") { + manage_stack.set_visible_child_name("main"); + cancel_button.label = _("Cancel"); + } + + if (manage_stack.get_visible_child_name() == "confirm") { + if (return_to_main) { + manage_stack.set_visible_child_name("main"); + cancel_button.label = _("Cancel"); + } else { + manage_stack.set_visible_child_name("verify"); + } + } + + ok_button.sensitive = false; + } + + private Box make_action_box(string title, string desc){ + Box box = new Box(Orientation.VERTICAL, 0) { visible = true, margin_start = 20, margin_end = 20, margin_top = 14, margin_bottom = 14 }; + Label lbl_title = new Label(title) { visible = true, halign = Align.START }; + Label lbl_desc = new Label(desc) { visible = true, xalign = 0, wrap = true, max_width_chars = 40 }; + + Pango.AttrList title_attrs = new Pango.AttrList(); + title_attrs.insert(Pango.attr_scale_new(1.1)); + lbl_title.attributes = title_attrs; + Pango.AttrList desc_attrs = new Pango.AttrList(); + desc_attrs.insert(Pango.attr_scale_new(0.8)); + lbl_desc.attributes = desc_attrs; + lbl_desc.add_css_class("dim-label"); + + box.append(lbl_title); + box.append(lbl_desc); + + return box; + } + + private void setup_main_screen() { + main_action_list.set_header_func((row, before_row) => { + if (row.get_header() == null && before_row != null) { + row.set_header(new Separator(Orientation.HORIZONTAL)); + } + }); + + ListBoxRow verify_row = new ListBoxRow() { visible = true }; + verify_row.set_child(make_action_box(_("Verify key fingerprint"), _("Compare this key's fingerprint with the fingerprint displayed on the contact's device."))); + ListBoxRow reject_row = new ListBoxRow() { visible = true }; + reject_row.set_child(make_action_box(_("Reject key"), _("Block encrypted communication with the contact's device that uses this key."))); + ListBoxRow accept_row = new ListBoxRow() {visible = true }; + accept_row.set_child(make_action_box(_("Accept key"), _("Allow encrypted communication with the contact's device that uses this key."))); + + switch((TrustLevel) device[db.identity_meta.trust_level]) { + case TrustLevel.TRUSTED: + main_desc_label.set_markup(_("This key is currently %s.").printf("<span color='#1A63D9'>"+_("accepted")+"</span>")+" "+_("This means it can be used by %s to receive and send encrypted messages.").printf(@"<b>$(device[db.identity_meta.address_name])</b>")); + main_action_list.append(verify_row); + main_action_list.append(reject_row); + break; + case TrustLevel.VERIFIED: + main_desc_label.set_markup(_("This key is currently %s.").printf("<span color='#1A63D9'>"+_("verified")+"</span>")+" "+_("This means it can be used by %s to receive and send encrypted messages.").printf(@"<b>$(device[db.identity_meta.address_name])</b>") + " " + _("Additionally it has been verified to match the key on the contact's device.")); + main_action_list.append(reject_row); + break; + case TrustLevel.UNTRUSTED: + main_desc_label.set_markup(_("This key is currently %s.").printf("<span color='#D91900'>"+_("rejected")+"</span>")+" "+_("This means it cannot be used by %s to decipher your messages, and you won't see messages encrypted with it.").printf(@"<b>$(device[db.identity_meta.address_name])</b>")); + main_action_list.append(accept_row); + break; + case TrustLevel.UNKNOWN: + default: + break; + } + + //Row clicked - go to appropriate screen + main_action_list.row_activated.connect((row) => { + if(row == verify_row) { + manage_stack.set_visible_child_name("verify"); + } else if (row == reject_row) { + confirm_image.set_from_icon_name("action-unavailable-symbolic"); + confirm_title_label.label = _("Reject key"); + confirm_desc_label.set_markup(_("You won't see encrypted messages from the device of %s that uses this key. Conversely, that device won't be able to decipher your messages anymore.").printf(@"<b>$(device[db.identity_meta.address_name])</b>")); + manage_stack.set_visible_child_name("confirm"); + ok_button.sensitive = true; + return_to_main = true; + current_response = TrustLevel.UNTRUSTED; + } else if (row == accept_row) { + confirm_image.set_from_icon_name("dino-check-plain-symbolic"); + confirm_title_label.label = _("Accept key"); + confirm_desc_label.set_markup(_("You will be able to exchange encrypted messages with the device of %s that uses this key.").printf(@"<b>$(device[db.identity_meta.address_name])</b>")); + manage_stack.set_visible_child_name("confirm"); + ok_button.sensitive = true; + return_to_main = true; + current_response = TrustLevel.TRUSTED; + } + cancel_button.label = _("Back"); + }); + + manage_stack.set_visible_child_name("main"); + } + + private void setup_verify_screen() { + verify_label.set_markup(fingerprint_markup(fingerprint_from_base64(device[db.identity_meta.identity_key_public_base64]))); + } +} + +} diff --git a/main/src/plugins/omemo/ui/own_notifications.vala b/main/src/plugins/omemo/ui/own_notifications.vala new file mode 100644 index 00000000..7ff10e30 --- /dev/null +++ b/main/src/plugins/omemo/ui/own_notifications.vala @@ -0,0 +1,35 @@ +using Dino.Entities; +using Xmpp; +using Gtk; + +namespace Dino.Plugins.Omemo { + +public class OwnNotifications { + + private StreamInteractor stream_interactor; + private Plugin plugin; + private Account account; + + public OwnNotifications (Plugin plugin, StreamInteractor stream_interactor, Account account) { + this.stream_interactor = (!)stream_interactor; + this.plugin = plugin; + this.account = account; + stream_interactor.module_manager.get_module(account, StreamModule.IDENTITY).bundle_fetched.connect_after((jid, device_id, bundle) => { + if (jid.equals(account.bare_jid) && plugin.has_new_devices(account, account.bare_jid)) { + display_notification(); + } + }); + + if (plugin.has_new_devices(account, account.bare_jid)) { + display_notification(); + } + } + + private void display_notification() { + Notification notification = new Notification(_("OMEMO trust decision required")); + notification.set_default_action_and_target_value("app.own-keys", new Variant.int32(account.id)); + notification.set_body(_("Did you add a new device for account %s?").printf(@"$(account.bare_jid.to_string())")); + plugin.app.send_notification(account.id.to_string()+"-new-device", notification); + } +} +} diff --git a/main/src/plugins/omemo/ui/util.vala b/main/src/plugins/omemo/ui/util.vala new file mode 100644 index 00000000..e250ff4d --- /dev/null +++ b/main/src/plugins/omemo/ui/util.vala @@ -0,0 +1,40 @@ +using Xmpp.Util; + +namespace Dino.Plugins.Omemo { + +public static string fingerprint_from_base64(string b64) { + uint8[] arr = Base64.decode(b64); + + arr = arr[1:arr.length]; + string s = ""; + foreach (uint8 i in arr) { + string tmp = i.to_string("%x"); + if (tmp.length == 1) tmp = "0" + tmp; + s = s + tmp; + } + + return s; +} + +public static string fingerprint_markup(string s) { + return "<span font_family='monospace' font='9'>" + format_fingerprint(s) + "</span>"; +} + +public static string format_fingerprint(string s) { + string markup = ""; + for (int i = 0; i < s.length; i += 4) { + string four_chars = s.substring(i, 4).down(); + + if (i % 32 == 0 && i != 0) markup += "\n"; + markup += four_chars; + if (i % 16 == 12 && i % 32 != 28) { + markup += " "; + } + if (i % 8 == 4 && i % 16 != 12) { + markup += "\u00a0"; // Non-breaking space + } + } + return markup; +} + +} diff --git a/main/src/plugins/openpgp/contact_details_provider.vala b/main/src/plugins/openpgp/contact_details_provider.vala new file mode 100644 index 00000000..353c0b85 --- /dev/null +++ b/main/src/plugins/openpgp/contact_details_provider.vala @@ -0,0 +1,51 @@ +using Gtk; + +using Dino.Entities; + +namespace Dino.Plugins.OpenPgp { + +public class ContactDetailsProvider : Plugins.ContactDetailsProvider, Object { + public string id { get { return "pgp_info"; } } + public string tab { get { return "encryption"; } } + + private StreamInteractor stream_interactor; + + public ContactDetailsProvider(StreamInteractor stream_interactor) { + this.stream_interactor = stream_interactor; + } + + public void populate(Conversation conversation, Plugins.ContactDetails contact_details, WidgetType type) { } + + public Object? get_widget(Conversation conversation) { + var preferences_group = new Adw.PreferencesGroup() { title="OpenPGP" }; + + if (conversation.type_ != Conversation.Type.CHAT) return null; + + string? key_id = stream_interactor.get_module(Manager.IDENTITY).get_key_id(conversation.account, conversation.counterpart); + if (key_id == null) return null; + + Gee.List<GPG.Key>? keys = null; + try { + keys = GPGHelper.get_keylist(key_id); + } catch (Error e) { } + + var str = ""; + if (keys != null && keys.size > 0) { + str = markup_id(keys[0].fpr, true); + } else { + str = _("Key not in keychain") + "\n" + markup_id(key_id, false); + } + + var view = new Adw.ActionRow() { + title = _("Fingerprint"), + subtitle = str, + subtitle_selectable = true + }; + + preferences_group.add(view); + + return preferences_group; + } +} + +} diff --git a/main/src/plugins/openpgp/database.vala b/main/src/plugins/openpgp/database.vala new file mode 100644 index 00000000..4c05571e --- /dev/null +++ b/main/src/plugins/openpgp/database.vala @@ -0,0 +1,137 @@ +using Dino.Entities; +using Dino.Qlite; +using Xmpp; + +namespace Dino.Plugins.OpenPgp { + +public class Database : Qlite.Database { + private const int VERSION = 1; + + public class AccountSetting : Table { + public Column<int> account_id = new Column.Integer("account_id") { primary_key = true }; + public Column<string> key = new Column.Text("key") { not_null = true }; + public Column<string> signed_data = new Column.Text("signed_data") { min_version = 1 }; + public Column<string> sig = new Column.Text("sig") { min_version = 1 }; + + internal AccountSetting(Database db) { + base(db, "account_setting"); + init({account_id, key, signed_data, sig}); + } + } + + public class ContactKey : Table { + public Column<string> jid = new Column.Text("jid") { primary_key = true }; + public Column<string> key = new Column.Text("key"); + public Column<string> sig = new Column.Text("sig") { min_version = 1 }; + public Column<string> signed_data = new Column.Text("signed_data") { min_version = 1 }; + + internal ContactKey(Database db) { + base(db, "contact_key"); + init({jid, key, sig, signed_data}); + } + + internal void migrate_from_v0(Database db) { + string column_list = ""; + foreach (Column c in columns) { + if (c.min_version <= VERSION && c.max_version >= VERSION) { + if (column_list == "") { + column_list = c.name; + } else { + column_list += ", " + c.name; + } + } + } + db.exec(@"ALTER TABLE $name RENAME TO _$(name)_1"); + create_table_at_version(VERSION); + db.exec(@"INSERT INTO $name ($column_list) SELECT $column_list FROM _$(name)_1"); + db.exec(@"DROP TABLE _$(name)_1"); + } + } + + public AccountSetting account_setting_table { get; private set; } + public ContactKey contact_key_table { get; private set; } + + public Database(string filename) { + base(filename, VERSION); + this.account_setting_table = new AccountSetting(this); + this.contact_key_table = new ContactKey(this); + init({account_setting_table, contact_key_table}); + + try { + exec("PRAGMA journal_mode = WAL"); + exec("PRAGMA synchronous = NORMAL"); + exec("PRAGMA secure_delete = ON"); + } catch (Error e) { + error("Failed to set OpenPGP database properties: %s", e.message); + } + } + + public void set_contact_signature(Jid jid, string sig, string signed_data) { + contact_key_table.upsert() + .value(contact_key_table.jid, jid.to_string(), true) + .value(contact_key_table.sig, sig) + .value(contact_key_table.signed_data, signed_data) + .value_null(contact_key_table.key) + .perform(); + } + + public void clear_contact_signature(Jid jid) { + contact_key_table.upsert() + .value(contact_key_table.jid, jid.to_string(), true) + .value_null(contact_key_table.sig) + .value_null(contact_key_table.signed_data) + .perform(); + } + + public void set_contact_key(Jid jid, string key) { + contact_key_table.upsert() + .value(contact_key_table.jid, jid.to_string(), true) + .value(contact_key_table.key, key) + .perform(); + } + + public string? get_contact_key(Jid jid) { + return get_contact_key_row(jid)[contact_key_table.key]; + } + + public RowOption get_contact_key_row(Jid jid) { + return contact_key_table.select({contact_key_table.key}) + .with(contact_key_table.jid, "=", jid.to_string()).row(); + } + + public void set_account_key(Account account, string key) { + account_setting_table.upsert() + .value(account_setting_table.account_id, account.id, true) + .value(account_setting_table.key, key) + .value_null(account_setting_table.signed_data) + .value_null(account_setting_table.sig) + .perform(); + } + + public void set_account_signature(Account account, string signed_data, string sig) { + account_setting_table.update() + .with(account_setting_table.account_id, "=", account.id) + .set(account_setting_table.signed_data, signed_data) + .set(account_setting_table.sig, sig) + .perform(); + } + + public string? get_account_signature(Account account, string signed_data) { + return account_setting_table.select({account_setting_table.sig}) + .with(account_setting_table.account_id, "=", account.id) + .with(account_setting_table.signed_data, "=", signed_data)[account_setting_table.sig]; + } + + public string? get_account_key(Account account) { + return account_setting_table.select({account_setting_table.key}) + .with(account_setting_table.account_id, "=", account.id)[account_setting_table.key]; + } + + public override void migrate(long oldVersion) { + if (oldVersion < 1) { + contact_key_table.migrate_from_v0(this); + } + } +} + +} diff --git a/main/src/plugins/openpgp/encryption_list_entry.vala b/main/src/plugins/openpgp/encryption_list_entry.vala new file mode 100644 index 00000000..cf5da8c4 --- /dev/null +++ b/main/src/plugins/openpgp/encryption_list_entry.vala @@ -0,0 +1,72 @@ +using Gee; +using Gtk; + +using Dino.Entities; +using Xmpp; + +namespace Dino.Plugins.OpenPgp { + +private class EncryptionListEntry : Plugins.EncryptionListEntry, Object { + + private StreamInteractor stream_interactor; + private Database db; + + public EncryptionListEntry(StreamInteractor stream_interactor, Database db) { + this.stream_interactor = stream_interactor; + this.db = db; + } + + public Entities.Encryption encryption { get { + return Encryption.PGP; + }} + + public string name { get { + return "OpenPGP"; + }} + + public Object? get_encryption_icon(Entities.Conversation conversation, ContentItem content_item) { + return null; + } + + public string? get_encryption_icon_name(Entities.Conversation conversation, ContentItem content_item) { + return null; + } + + public void encryption_activated(Entities.Conversation conversation, Plugins.SetInputFieldStatus input_status_callback) { + try { + GPGHelper.get_public_key(db.get_account_key(conversation.account) ?? ""); + } catch (Error e) { + input_status_callback(new Plugins.InputFieldStatus("You didn't configure OpenPGP for this account. You can do that in the Accounts Dialog.", Plugins.InputFieldStatus.MessageType.ERROR, Plugins.InputFieldStatus.InputState.NO_SEND)); + return; + } + + if (conversation.type_ == Conversation.Type.CHAT) { + string? key_id = stream_interactor.get_module(Manager.IDENTITY).get_key_id(conversation.account, conversation.counterpart); + if (key_id == null) { + input_status_callback(new Plugins.InputFieldStatus("This contact does not support %s encryption.".printf("OpenPGP"), Plugins.InputFieldStatus.MessageType.ERROR, Plugins.InputFieldStatus.InputState.NO_SEND)); + return; + } + try { + GPGHelper.get_keylist(key_id); + } catch (Error e) { + input_status_callback(new Plugins.InputFieldStatus("This contact's OpenPGP key is not in your keyring.", Plugins.InputFieldStatus.MessageType.ERROR, Plugins.InputFieldStatus.InputState.NO_SEND)); + } + } else if (conversation.type_ == Conversation.Type.GROUPCHAT) { + Gee.List<Jid> muc_jids = new Gee.ArrayList<Jid>(); + Gee.List<Jid>? occupants = stream_interactor.get_module(MucManager.IDENTITY).get_occupants(conversation.counterpart, conversation.account); + if (occupants != null) muc_jids.add_all(occupants); + Gee.List<Jid>? offline_members = stream_interactor.get_module(MucManager.IDENTITY).get_offline_members(conversation.counterpart, conversation.account); + if (offline_members != null) muc_jids.add_all(offline_members); + + foreach (Jid jid in muc_jids) { + string? key_id = stream_interactor.get_module(Manager.IDENTITY).get_key_id(conversation.account, jid); + if (key_id == null) { + input_status_callback(new Plugins.InputFieldStatus("A member's OpenPGP key is not in your keyring: %s / %s.".printf(jid.to_string(), key_id), Plugins.InputFieldStatus.MessageType.ERROR, Plugins.InputFieldStatus.InputState.NO_SEND)); + return; + } + } + } + } +} + +} diff --git a/main/src/plugins/openpgp/encryption_preferences_entry.vala b/main/src/plugins/openpgp/encryption_preferences_entry.vala new file mode 100644 index 00000000..b03f6acd --- /dev/null +++ b/main/src/plugins/openpgp/encryption_preferences_entry.vala @@ -0,0 +1,82 @@ +using Adw; +using Dino.Entities; +using Gtk; + +namespace Dino.Plugins.OpenPgp { + + public class PgpPreferencesEntry : Plugins.EncryptionPreferencesEntry { + + private Plugin plugin; + + public PgpPreferencesEntry(Plugin plugin) { + this.plugin = plugin; + } + + public override Object? get_widget(Account account, WidgetType type) { + if (type != WidgetType.GTK4) return null; + StringList string_list = new StringList(null); + string_list.append(_("Querying GnuPG")); + + Adw.PreferencesGroup preferences_group = new Adw.PreferencesGroup() { title="OpenPGP" }; + populate_string_list.begin(account, preferences_group); + + return preferences_group; + } + + public override string id { get { return "pgp_preferences_encryption"; }} + + private async void populate_string_list(Account account, Adw.PreferencesGroup preferences_group) { + var keys = yield get_pgp_keys(); + + if (keys == null) { + preferences_group.add(new Adw.ActionRow() { title=_("Announce key"), subtitle=_("Error in GnuPG") }); + return; + } + if (keys.size == 0) { + preferences_group.add(new Adw.ActionRow() { title=_("Announce key"), subtitle= _("No keys available. Generate one!") }); + return; + } + + StringList string_list = new StringList(null); + var drop_down = new Adw.ComboRow() { title = _("Announce key") }; + drop_down.model = string_list; + preferences_group.add(drop_down); + + string_list.append(_("Disabled")); + for (int i = 0; i < keys.size; i++) { + string_list.append(@"$(keys[i].uids[0].uid)\n$(keys[i].fpr.substring(24, 16))"); + if (keys[i].fpr == plugin.db.get_account_key(account)) { + drop_down.selected = i + 1; + } + } + + drop_down.notify["selected"].connect(() => { + var key_id = drop_down.selected == 0 ? "" : keys[(int)drop_down.selected - 1].fpr; + string? signed_data = null, sig = null; + if (plugin.modules.has_key(account)) { + plugin.modules[account].set_private_key_id(key_id, ref signed_data, ref sig); + } + plugin.db.set_account_key(account, key_id); + if (signed_data != null && sig != null) { + plugin.db.set_account_signature(account, signed_data, sig); + } + }); + } + + private static async Gee.List<GPG.Key> get_pgp_keys() { + Gee.List<GPG.Key> keys = null; + SourceFunc callback = get_pgp_keys.callback; + new Thread<void*> (null, () => { // Querying GnuPG might take some time + try { + keys = GPGHelper.get_keylist(null, true); + } catch (Error e) { + warning(e.message); + } + Idle.add((owned)callback); + return null; + }); + yield; + return keys; + } + } +}
\ No newline at end of file diff --git a/main/src/plugins/openpgp/file_transfer/file_decryptor.vala b/main/src/plugins/openpgp/file_transfer/file_decryptor.vala new file mode 100644 index 00000000..2deb5823 --- /dev/null +++ b/main/src/plugins/openpgp/file_transfer/file_decryptor.vala @@ -0,0 +1,50 @@ +using Dino.Entities; + +namespace Dino.Plugins.OpenPgp { + +public class PgpFileDecryptor : FileDecryptor, Object { + + public Encryption get_encryption() { + return Encryption.PGP; + } + + public FileReceiveData prepare_get_meta_info(Conversation conversation, FileTransfer file_transfer, FileReceiveData receive_data) { + return receive_data; + } + + public FileMeta prepare_download_file(Conversation conversation, FileTransfer file_transfer, FileReceiveData receive_data, FileMeta file_meta) { + return file_meta; + } + + public bool can_decrypt_file(Conversation conversation, FileTransfer file_transfer, FileReceiveData receive_data) { + return file_transfer.file_name.has_suffix("pgp") || + (file_transfer.content_type != null && file_transfer.content_type.get_mime_type() == "application/pgp-encrypted"); + } + + public async InputStream decrypt_file(InputStream encrypted_stream, Conversation conversation, FileTransfer file_transfer, FileReceiveData receive_data) throws FileReceiveError { + try { + uint8[] buf = new uint8[256]; + ByteArray data = new ByteArray(); + size_t len = -1; + do { + len = yield encrypted_stream.read_async(buf); + data.append(buf[0:len]); + } while(len > 0); + + GPGHelper.DecryptedData clear_data = GPGHelper.decrypt_data(data.data); + file_transfer.encryption = Encryption.PGP; + if (clear_data.filename != null && clear_data.filename != "") { + debug("Decrypting file %s from %s", clear_data.filename, file_transfer.file_name); + file_transfer.file_name = clear_data.filename; + } else if (file_transfer.file_name.has_suffix(".pgp")) { + debug("Decrypting file %s from %s", file_transfer.file_name.substring(0, file_transfer.file_name.length - 4), file_transfer.file_name); + file_transfer.file_name = file_transfer.file_name.substring(0, file_transfer.file_name.length - 4); + } + return new MemoryInputStream.from_data(clear_data.data, GLib.free); + } catch (Error e) { + throw new FileReceiveError.DECRYPTION_FAILED("PGP file decryption error: %s".printf(e.message)); + } + } +} + +} diff --git a/main/src/plugins/openpgp/file_transfer/file_encryptor.vala b/main/src/plugins/openpgp/file_transfer/file_encryptor.vala new file mode 100644 index 00000000..66e93bd9 --- /dev/null +++ b/main/src/plugins/openpgp/file_transfer/file_encryptor.vala @@ -0,0 +1,45 @@ +using Dino.Entities; + +namespace Dino.Plugins.OpenPgp { + +public class PgpFileEncryptor : Dino.FileEncryptor, Object { + + StreamInteractor stream_interactor; + + public PgpFileEncryptor(StreamInteractor stream_interactor) { + this.stream_interactor = stream_interactor; + } + + public bool can_encrypt_file(Conversation conversation, FileTransfer file_transfer) { + return conversation.encryption == Encryption.PGP; + } + + public FileMeta encrypt_file(Conversation conversation, FileTransfer file_transfer) throws FileSendError { + FileMeta file_meta = new FileMeta(); + + try { + GPG.Key[] keys = stream_interactor.get_module(Manager.IDENTITY).get_key_fprs(conversation); + uint8[] enc_content = GPGHelper.encrypt_file(file_transfer.get_file().get_path(), keys, GPG.EncryptFlags.ALWAYS_TRUST, file_transfer.file_name); + file_transfer.input_stream = new MemoryInputStream.from_data(enc_content, GLib.free); + file_transfer.encryption = Encryption.PGP; + file_transfer.server_file_name = Xmpp.random_uuid() + ".pgp"; + file_meta.size = enc_content.length; + } catch (Error e) { + throw new FileSendError.ENCRYPTION_FAILED("PGP file encryption error: %s".printf(e.message)); + } + debug("Encrypting file %s as %s", file_transfer.file_name, file_transfer.server_file_name); + + return file_meta; + } + + public FileSendData? preprocess_send_file(Conversation conversation, FileTransfer file_transfer, FileSendData file_send_data, FileMeta file_meta) { + HttpFileSendData? send_data = file_send_data as HttpFileSendData; + if (send_data == null) return null; + + send_data.encrypt_message = false; + + return file_send_data; + } +} + +} diff --git a/main/src/plugins/openpgp/gpgme_fix.c b/main/src/plugins/openpgp/gpgme_fix.c new file mode 100644 index 00000000..a351bcbc --- /dev/null +++ b/main/src/plugins/openpgp/gpgme_fix.c @@ -0,0 +1,12 @@ +#include <src/plugins/openpgp/gpgme_fix.h> + +GRecMutex gpgme_global_mutex = {0}; + +gpgme_key_t gpgme_key_ref_vapi (gpgme_key_t key) { + gpgme_key_ref(key); + return key; +} +gpgme_key_t gpgme_key_unref_vapi (gpgme_key_t key) { + gpgme_key_unref(key); + return key; +} diff --git a/main/src/plugins/openpgp/gpgme_fix.h b/main/src/plugins/openpgp/gpgme_fix.h new file mode 100644 index 00000000..714614fc --- /dev/null +++ b/main/src/plugins/openpgp/gpgme_fix.h @@ -0,0 +1,12 @@ +#ifndef GPGME_FIX +#define GPGME_FIX 1 + +#include <glib.h> +#include <gpgme.h> + +extern GRecMutex gpgme_global_mutex; + +gpgme_key_t gpgme_key_ref_vapi (gpgme_key_t key); +gpgme_key_t gpgme_key_unref_vapi (gpgme_key_t key); + +#endif diff --git a/main/src/plugins/openpgp/gpgme_helper.vala b/main/src/plugins/openpgp/gpgme_helper.vala new file mode 100644 index 00000000..18d07c06 --- /dev/null +++ b/main/src/plugins/openpgp/gpgme_helper.vala @@ -0,0 +1,185 @@ +using Gee; +using GPG; + +namespace GPGHelper { + +private static bool initialized = false; + +public static string encrypt_armor(string plain, Key[] keys, EncryptFlags flags) throws GLib.Error { + global_mutex.lock(); + try { + initialize(); + Data plain_data = Data.create_from_memory(plain.data, false); + Context context = Context.create(); + context.set_armor(true); + Data enc_data = context.op_encrypt(keys, flags, plain_data); + return get_string_from_data(enc_data); + } finally { + global_mutex.unlock(); + } +} + +public static uint8[] encrypt_file(string uri, Key[] keys, EncryptFlags flags, string file_name) throws GLib.Error { + global_mutex.lock(); + try { + initialize(); + Data plain_data = Data.create_from_file(uri); + plain_data.set_file_name(file_name); + Context context = Context.create(); + context.set_armor(true); + Data enc_data = context.op_encrypt(keys, flags, plain_data); + return get_uint8_from_data(enc_data); + } finally { + global_mutex.unlock(); + } +} + +public static string decrypt(string encr) throws GLib.Error { + global_mutex.lock(); + try { + initialize(); + Data enc_data = Data.create_from_memory(encr.data, false); + Context context = Context.create(); + Data dec_data = context.op_decrypt(enc_data); + return get_string_from_data(dec_data); + } finally { + global_mutex.unlock(); + } +} + +public class DecryptedData { + public uint8[] data { get; set; } + public string filename { get; set; } +} + +public static DecryptedData decrypt_data(uint8[] data) throws GLib.Error { + global_mutex.lock(); + try { + initialize(); + Data enc_data = Data.create_from_memory(data, false); + Context context = Context.create(); + Data dec_data = context.op_decrypt(enc_data); + DecryptResult* dec_res = context.op_decrypt_result(); + return new DecryptedData() { data=get_uint8_from_data(dec_data), filename=dec_res->file_name}; + } finally { + global_mutex.unlock(); + } +} + +public static string sign(string plain, SigMode mode, Key? key = null) throws GLib.Error { + global_mutex.lock(); + try { + initialize(); + Data plain_data = Data.create_from_memory(plain.data, false); + Context context = Context.create(); + if (key != null) context.signers_add(key); + Data signed_data = context.op_sign(plain_data, mode); + return get_string_from_data(signed_data); + } finally { + global_mutex.unlock(); + } +} + +public static string? get_sign_key(string signature, string? text) throws GLib.Error { + global_mutex.lock(); + try { + initialize(); + Data sig_data = Data.create_from_memory(signature.data, false); + Data text_data; + if (text != null) { + text_data = Data.create_from_memory(text.data, false); + } else { + text_data = Data.create(); + } + Context context = Context.create(); + context.op_verify(sig_data, text_data); + VerifyResult* verify_res = context.op_verify_result(); + if (verify_res == null || verify_res.signatures == null) return null; + return verify_res.signatures.fpr; + } finally { + global_mutex.unlock(); + } +} + +public static Gee.List<Key> get_keylist(string? pattern = null, bool secret_only = false) throws GLib.Error { + global_mutex.lock(); + try { + initialize(); + + Gee.List<Key> keys = new ArrayList<Key>(); + Context context = Context.create(); + context.op_keylist_start(pattern, secret_only ? 1 : 0); + try { + while (true) { + Key key = context.op_keylist_next(); + keys.add(key); + } + } catch (Error e) { + if (e.code != GPGError.ErrorCode.EOF) throw e; + } + context.op_keylist_end(); + return keys; + } finally { + global_mutex.unlock(); + } +} + +public static Key? get_public_key(string sig) throws GLib.Error { + return get_key(sig, false); +} + +public static Key? get_private_key(string sig) throws GLib.Error { + return get_key(sig, true); +} + +private static Key? get_key(string sig, bool priv) throws GLib.Error { + global_mutex.lock(); + try { + initialize(); + Context context = Context.create(); + Key key = context.get_key(sig, priv); + return key; + } finally { + global_mutex.unlock(); + } +} + +private static string get_string_from_data(Data data) { + const size_t BUF_SIZE = 256; + data.seek(0); + uint8[] buf = new uint8[BUF_SIZE + 1]; + ssize_t len = 0; + string res = ""; + do { + len = data.read(buf, BUF_SIZE); + if (len > 0) { + buf[len] = 0; + res += (string) buf; + } + } while (len > 0); + return res; +} + +private static uint8[] get_uint8_from_data(Data data) { + const size_t BUF_SIZE = 256; + data.seek(0); + uint8[] buf = new uint8[BUF_SIZE + 1]; + ssize_t len = 0; + ByteArray res = new ByteArray(); + do { + len = data.read(buf, BUF_SIZE); + if (len > 0) { + res.append(buf[0:len]); + } + } while (len > 0); + return res.data; +} + +private static void initialize() { + if (!initialized) { + check_version(); + initialized = true; + } +} + +} diff --git a/main/src/plugins/openpgp/manager.vala b/main/src/plugins/openpgp/manager.vala new file mode 100644 index 00000000..23cc93c2 --- /dev/null +++ b/main/src/plugins/openpgp/manager.vala @@ -0,0 +1,123 @@ +using Gee; +using Xmpp; + +using Xmpp; +using Dino.Entities; + +namespace Dino.Plugins.OpenPgp { + +public class Manager : StreamInteractionModule, Object { + public static ModuleIdentity<Manager> IDENTITY = new ModuleIdentity<Manager>("pgp_manager"); + public string id { get { return IDENTITY.id; } } + + public const string MESSAGE_ENCRYPTED = "pgp"; + + private StreamInteractor stream_interactor; + private Database db; + private ReceivedMessageListener received_message_listener = new ReceivedMessageListener(); + + public static void start(StreamInteractor stream_interactor, Database db) { + Manager m = new Manager(stream_interactor, db); + stream_interactor.add_module(m); + } + + private Manager(StreamInteractor stream_interactor, Database db) { + this.stream_interactor = stream_interactor; + this.db = db; + + stream_interactor.account_added.connect(on_account_added); + stream_interactor.get_module(MessageProcessor.IDENTITY).received_pipeline.connect(received_message_listener); + stream_interactor.get_module(MessageProcessor.IDENTITY).pre_message_send.connect(check_encypt); + } + + public GPG.Key[] get_key_fprs(Conversation conversation) throws Error { + Gee.List<string> keys = new Gee.ArrayList<string>(); + keys.add(db.get_account_key(conversation.account)); + if (conversation.type_ == Conversation.Type.GROUPCHAT) { + Gee.List<Jid> muc_jids = new Gee.ArrayList<Jid>(); + Gee.List<Jid>? occupants = stream_interactor.get_module(MucManager.IDENTITY).get_occupants(conversation.counterpart, conversation.account); + if (occupants != null) muc_jids.add_all(occupants); + Gee.List<Jid>? offline_members = stream_interactor.get_module(MucManager.IDENTITY).get_offline_members(conversation.counterpart, conversation.account); + if (occupants != null) muc_jids.add_all(offline_members); + + foreach (Jid jid in muc_jids) { + string? key_id = stream_interactor.get_module(Manager.IDENTITY).get_key_id(conversation.account, jid); + if (key_id != null && GPGHelper.get_keylist(key_id).size > 0 && !keys.contains(key_id)) { + keys.add(key_id); + } + } + } else { + string? key_id = get_key_id(conversation.account, conversation.counterpart); + if (key_id != null) { + keys.add(key_id); + } + } + GPG.Key[] gpgkeys = new GPG.Key[keys.size]; + for (int i = 0; i < keys.size; i++) { + try { + GPG.Key key = GPGHelper.get_public_key(keys[i]); + if (key != null) gpgkeys[i] = key; + } catch (Error e) {} + } + + return gpgkeys; + } + + private void check_encypt(Entities.Message message, Xmpp.MessageStanza message_stanza, Conversation conversation) { + try { + if (message.encryption == Encryption.PGP) { + GPG.Key[] keys = get_key_fprs(conversation); + XmppStream? stream = stream_interactor.get_stream(conversation.account); + if (stream != null) { + bool encrypted = stream.get_module(Module.IDENTITY).encrypt(message_stanza, keys); + if (!encrypted) message.marked = Entities.Message.Marked.WONTSEND; + } + } + } catch (Error e) { + message.marked = Entities.Message.Marked.WONTSEND; + } + } + + public string? get_key_id(Account account, Jid jid) { + Jid search_jid = stream_interactor.get_module(MucManager.IDENTITY).is_groupchat_occupant(jid, account) ? jid : jid.bare_jid; + Qlite.RowOption key_row = db.get_contact_key_row(search_jid); + if (!key_row.is_present()) return null; + string? key_id = key_row.get(db.contact_key_table.key); + if (key_id != null) return key_id; + string? sig = key_row.get(db.contact_key_table.sig); + string? signed_data = key_row.get(db.contact_key_table.signed_data); + if (sig == null || signed_data == null) return null; + key_id = get_sign_key(sig, signed_data); + if (key_id != null) { + db.set_contact_key(search_jid, key_id); + } + return key_id; + } + + private void on_account_added(Account account) { + stream_interactor.module_manager.get_module(account, Module.IDENTITY).received_jid_presence_signature.connect((stream, jid, sig, signed_data) => { + on_jid_signature_received(account, jid, sig, signed_data); + }); + } + + private void on_jid_signature_received(Account account, Jid jid, string sig, string signed_data) { + Jid set_jid = stream_interactor.get_module(MucManager.IDENTITY).is_groupchat_occupant(jid, account) ? jid : jid.bare_jid; + db.set_contact_signature(set_jid, sig, signed_data); + } + + private class ReceivedMessageListener : MessageListener { + + public string[] after_actions_const = new string[]{ }; + public override string action_group { get { return "DECRYPT"; } } + public override string[] after_actions { get { return after_actions_const; } } + + public override async bool run(Entities.Message message, Xmpp.MessageStanza stanza, Conversation conversation) { + if (MessageFlag.get_flag(stanza) != null && MessageFlag.get_flag(stanza).decrypted) { + message.encryption = Encryption.PGP; + } + return false; + } + } +} + +} diff --git a/main/src/plugins/openpgp/plugin.vala b/main/src/plugins/openpgp/plugin.vala new file mode 100644 index 00000000..67b94455 --- /dev/null +++ b/main/src/plugins/openpgp/plugin.vala @@ -0,0 +1,47 @@ +using Gee; + +using Dino.Entities; + +namespace Dino.Plugins.OpenPgp { + +public class Plugin : Plugins.RootInterface, Object { + public Dino.Application app; + public Database db; + public HashMap<Account, Module> modules = new HashMap<Account, Module>(Account.hash_func, Account.equals_func); + + private EncryptionListEntry list_entry; + private ContactDetailsProvider contact_details_provider; + + public void registered(Dino.Application app) { + this.app = app; + this.db = new Database(Path.build_filename(Application.get_storage_dir(), "pgp.db")); + this.list_entry = new EncryptionListEntry(app.stream_interactor, db); + this.contact_details_provider = new ContactDetailsProvider(app.stream_interactor); + + app.plugin_registry.register_encryption_list_entry(list_entry); + app.plugin_registry.register_encryption_preferences_entry(new PgpPreferencesEntry(this)); + app.plugin_registry.register_contact_details_entry(contact_details_provider); + app.stream_interactor.module_manager.initialize_account_modules.connect(on_initialize_account_modules); + + Manager.start(app.stream_interactor, db); + app.stream_interactor.get_module(FileManager.IDENTITY).add_file_encryptor(new PgpFileEncryptor(app.stream_interactor)); + app.stream_interactor.get_module(FileManager.IDENTITY).add_file_decryptor(new PgpFileDecryptor()); + JingleFileHelperRegistry.instance.add_encryption_helper(Encryption.PGP, new JingleFileEncryptionHelperTransferOnly()); + } + + public void shutdown() { } + + private void on_initialize_account_modules(Account account, ArrayList<Xmpp.XmppStreamModule> modules) { + string signed_data = ""; + string? sig = db.get_account_signature(account, signed_data); + bool sig_was_unknown = sig == null; + Module module = new Module(db.get_account_key(account), ref signed_data, ref sig); + if (sig_was_unknown && sig != null) { + db.set_account_signature(account, signed_data, sig); + } + this.modules[account] = module; + modules.add(module); + } +} + +} diff --git a/main/src/plugins/openpgp/stream_module.vala b/main/src/plugins/openpgp/stream_module.vala new file mode 100644 index 00000000..9985df26 --- /dev/null +++ b/main/src/plugins/openpgp/stream_module.vala @@ -0,0 +1,159 @@ +using GPG; + +using Xmpp; + +namespace Dino.Plugins.OpenPgp { + private const string NS_URI = "jabber:x"; + private const string NS_URI_ENCRYPTED = NS_URI + ":encrypted"; + private const string NS_URI_SIGNED = NS_URI + ":signed"; + + public class Module : XmppStreamModule { + public static Xmpp.ModuleIdentity<Module> IDENTITY = new Xmpp.ModuleIdentity<Module>(NS_URI, "0027_current_pgp_usage"); + + public signal void received_jid_presence_signature(XmppStream stream, Jid jid, string sig, string signed_data); + + private string? signed_status = null; + private Key? own_key = null; + private ReceivedPipelineDecryptListener received_pipeline_decrypt_listener = new ReceivedPipelineDecryptListener(); + + public Module(string? own_key_id = null, ref string? signed_data, ref string? sig) { + set_private_key_id(own_key_id, ref signed_data, ref sig); + } + + public void set_private_key_id(string? own_key_id, ref string? signed_data, ref string? sig) { + if (own_key_id != null && sig == null) { + try { + own_key = GPGHelper.get_private_key(own_key_id); + if (own_key == null) warning("Can't get PGP private key"); + } catch (Error e) { } + if (own_key != null) { + if (signed_data == null) signed_data = ""; + sig = gpg_sign(signed_data, own_key); + } + } + signed_status = own_key_id != null ? sig : null; + } + + public bool encrypt(MessageStanza message, GPG.Key[] keys) { + string? enc_body = gpg_encrypt(message.body, keys); + if (enc_body != null) { + message.stanza.put_node(new StanzaNode.build("x", NS_URI_ENCRYPTED).add_self_xmlns().put_node(new StanzaNode.text(enc_body))); + message.body = "[This message is OpenPGP encrypted (see XEP-0027)]"; + Xep.ExplicitEncryption.add_encryption_tag_to_message(message, NS_URI_ENCRYPTED); + return true; + } + return false; + } + + public override void attach(XmppStream stream) { + stream.get_module(Presence.Module.IDENTITY).received_presence.connect(on_received_presence); + stream.get_module(Presence.Module.IDENTITY).pre_send_presence_stanza.connect(on_pre_send_presence_stanza); + stream.get_module(MessageModule.IDENTITY).received_pipeline.connect(received_pipeline_decrypt_listener); + } + + public override void detach(XmppStream stream) { + stream.get_module(Presence.Module.IDENTITY).received_presence.disconnect(on_received_presence); + stream.get_module(Presence.Module.IDENTITY).pre_send_presence_stanza.disconnect(on_pre_send_presence_stanza); + stream.get_module(MessageModule.IDENTITY).received_pipeline.disconnect(received_pipeline_decrypt_listener); + } + + public override string get_ns() { return NS_URI; } + public override string get_id() { return IDENTITY.id; } + + private void on_received_presence(XmppStream stream, Presence.Stanza presence) { + StanzaNode x_node = presence.stanza.get_subnode("x", NS_URI_SIGNED); + if (x_node == null) return; + string? sig = x_node.get_string_content(); + if (sig == null) return; + string signed_data = presence.status == null ? "" : presence.status; + received_jid_presence_signature(stream, presence.from, sig, signed_data); + } + + private void on_pre_send_presence_stanza(XmppStream stream, Presence.Stanza presence) { + if (presence.type_ == Presence.Stanza.TYPE_AVAILABLE && signed_status != null) { + presence.stanza.put_node(new StanzaNode.build("x", NS_URI_SIGNED).add_self_xmlns().put_node(new StanzaNode.text(signed_status))); + } + } + + private static string? gpg_encrypt(string plain, GPG.Key[] keys) { + string encr; + try { + encr = GPGHelper.encrypt_armor(plain, keys, GPG.EncryptFlags.ALWAYS_TRUST); + } catch (Error e) { + return null; + } + int encryption_start = encr.index_of("\n\n") + 2; + return encr.substring(encryption_start, encr.length - "\n-----END PGP MESSAGE-----".length - encryption_start); + } + + private static string? gpg_sign(string str, Key key) { + string signed; + try { + signed = GPGHelper.sign(str, GPG.SigMode.CLEAR, key); + } catch (Error e) { + return null; + } + int signature_start = signed.index_of("-----BEGIN PGP SIGNATURE-----"); + signature_start = signed.index_of("\n\n", signature_start) + 2; + return signed.substring(signature_start, signed.length - "\n-----END PGP SIGNATURE-----".length - signature_start); + } + } + + public class MessageFlag : Xmpp.MessageFlag { + public const string id = "pgp"; + + public bool decrypted = false; + + public static MessageFlag? get_flag(MessageStanza message) { + return (MessageFlag) message.get_flag(NS_URI, id); + } + + public override string get_ns() { return NS_URI; } + public override string get_id() { return id; } + } + +public class ReceivedPipelineDecryptListener : StanzaListener<MessageStanza> { + + private string[] after_actions_const = {"MODIFY_BODY"}; + + public override string action_group { get { return "ENCRYPT_BODY"; } } + public override string[] after_actions { get { return after_actions_const; } } + + public override async bool run(XmppStream stream, MessageStanza message) { + string? encrypted = get_cyphertext(message); + if (encrypted != null) { + MessageFlag flag = new MessageFlag(); + message.add_flag(flag); + string? decrypted = yield gpg_decrypt(encrypted); + if (decrypted != null) { + flag.decrypted = true; + message.body = decrypted; + } + } + return false; + } + + private static async string? gpg_decrypt(string enc) { + SourceFunc callback = gpg_decrypt.callback; + string? res = null; + new Thread<void*> (null, () => { + string armor = "-----BEGIN PGP MESSAGE-----\n\n" + enc + "\n-----END PGP MESSAGE-----"; + try { + res = GPGHelper.decrypt(armor); + } catch (Error e) { + res = null; + } + Idle.add((owned) callback); + return null; + }); + yield; + return res; + } + + private string? get_cyphertext(MessageStanza message) { + StanzaNode? x_node = message.stanza.get_subnode("x", NS_URI_ENCRYPTED); + return x_node == null ? null : x_node.get_string_content(); + } +} + +} diff --git a/main/src/plugins/openpgp/util.vala b/main/src/plugins/openpgp/util.vala new file mode 100644 index 00000000..3c7a7594 --- /dev/null +++ b/main/src/plugins/openpgp/util.vala @@ -0,0 +1,29 @@ +using Gtk; + +using Dino.Entities; +using Xmpp.Util; + +namespace Dino.Plugins.OpenPgp { + +public static string markup_id(string s, bool is_fingerprint) { + string markup = is_fingerprint ? "" : "0x"; + for (int i = 0; i < s.length; i += 4) { + string four_chars = s.substring(i, 4).down(); + + if (i == 4 * 5) markup += "\n"; + markup += four_chars; + if (is_fingerprint) markup += " "; + } + return "<span font_family='monospace' font='9'>" + markup + "</span>"; +} + +private static string? get_sign_key(string sig, string signed_text) { + string armor = "-----BEGIN PGP MESSAGE-----\n\n" + sig + "\n-----END PGP MESSAGE-----"; + string? sign_key = null; + try { + sign_key = GPGHelper.get_sign_key(armor, signed_text); + } catch (Error e) { } + return sign_key; +} + +} diff --git a/main/src/plugins/rtp/codec_util.vala b/main/src/plugins/rtp/codec_util.vala new file mode 100644 index 00000000..9f21f5a6 --- /dev/null +++ b/main/src/plugins/rtp/codec_util.vala @@ -0,0 +1,418 @@ +using Gee; +using Xmpp; +using Xmpp.Xep; + +public class Dino.Plugins.Rtp.CodecUtil { + private Set<string> supported_elements = new HashSet<string>(); + private Set<string> unsupported_elements = new HashSet<string>(); + + public static Gst.Caps get_caps(string media, JingleRtp.PayloadType payload_type, bool incoming) { + Gst.Caps caps = new Gst.Caps.simple("application/x-rtp", + "media", typeof(string), media, + "payload", typeof(int), payload_type.id); + //"channels", typeof(int), payloadType.channels, + //"max-ptime", typeof(int), payloadType.maxptime); + unowned Gst.Structure s = caps.get_structure(0); + if (payload_type.clockrate != 0) { + s.set("clock-rate", typeof(int), payload_type.clockrate); + } + if (payload_type.name != null) { + s.set("encoding-name", typeof(string), payload_type.name.up()); + } + if (incoming) { + foreach (JingleRtp.RtcpFeedback rtcp_fb in payload_type.rtcp_fbs) { + if (rtcp_fb.subtype == null) { + s.set(@"rtcp-fb-$(rtcp_fb.type_)", typeof(bool), true); + } else { + s.set(@"rtcp-fb-$(rtcp_fb.type_)-$(rtcp_fb.subtype)", typeof(bool), true); + } + } + } + return caps; + } + + public static string? get_codec_from_payload(string media, JingleRtp.PayloadType payload_type) { + if (payload_type.name != null) return payload_type.name.down(); + if (media == "audio") { + switch (payload_type.id) { + case 0: + return "pcmu"; + case 8: + return "pcma"; + } + } + return null; + } + + public static string? get_media_type_from_payload(string media, JingleRtp.PayloadType payload_type) { + return get_media_type(media, get_codec_from_payload(media, payload_type)); + } + + public static string? get_media_type(string media, string? codec) { + if (codec == null) return null; + if (media == "audio") { + switch (codec) { + case "pcma": + return "audio/x-alaw"; + case "pcmu": + return "audio/x-mulaw"; + } + } + return @"$media/x-$codec"; + } + + public static string? get_rtp_pay_element_name_from_payload(string media, JingleRtp.PayloadType payload_type) { + return get_pay_candidate(media, get_codec_from_payload(media, payload_type)); + } + + public static string? get_pay_candidate(string media, string? codec) { + if (codec == null) return null; + return @"rtp$(codec)pay"; + } + + public static string? get_rtp_depay_element_name_from_payload(string media, JingleRtp.PayloadType payload_type) { + return get_depay_candidate(media, get_codec_from_payload(media, payload_type)); + } + + public static string? get_depay_candidate(string media, string? codec) { + if (codec == null) return null; + return @"rtp$(codec)depay"; + } + + public static string[] get_encode_candidates(string media, string? codec) { + if (codec == null) return new string[0]; + if (media == "audio") { + switch (codec) { + case "opus": + return new string[] {"opusenc"}; + case "speex": + return new string[] {"speexenc"}; + case "pcma": + return new string[] {"alawenc"}; + case "pcmu": + return new string[] {"mulawenc"}; + case "g722": + return new string[] {"avenc_g722"}; + } + } else if (media == "video") { + switch (codec) { + case "h264": + return new string[] { +#if ENABLE_MSDK + "msdkh264enc", +#endif +#if ENABLE_VAAPI + "vaapih264enc", +#endif + "x264enc" + }; + case "vp9": + return new string[] { +#if ENABLE_MSDK + "msdkvp9enc", +#endif +#if ENABLE_VAAPI + "vaapivp9enc", +#endif + "vp9enc" + }; + case "vp8": + return new string[] { +#if ENABLE_MSDK + "msdkvp8enc", +#endif +#if ENABLE_VAAPI + "vaapivp8enc", +#endif + "vp8enc" + }; + } + } + return new string[0]; + } + + public static string[] get_decode_candidates(string media, string? codec) { + if (codec == null) return new string[0]; + if (media == "audio") { + switch (codec) { + case "opus": + return new string[] {"opusdec"}; + case "speex": + return new string[] {"speexdec"}; + case "pcma": + return new string[] {"alawdec"}; + case "pcmu": + return new string[] {"mulawdec"}; + case "g722": + return new string[] {"avdec_g722"}; + } + } else if (media == "video") { + switch (codec) { + case "h264": + return new string[] { +#if ENABLE_MSDK + "msdkh264dec", +#endif +#if ENABLE_VAAPI + "vaapih264dec", +#endif + null + }; + case "vp9": + return new string[] { +#if ENABLE_MSDK + "msdkvp9dec", +#endif +#if ENABLE_VAAPI + "vaapivp9dec", +#endif + "vp9dec" + }; + case "vp8": + return new string[] { +#if ENABLE_MSDK + "msdkvp8dec", +#endif +#if ENABLE_VAAPI + "vaapivp8dec", +#endif + "vp8dec" + }; + } + } + return new string[0]; + } + + public static string? get_encode_prefix(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { + if (encode == "msdkh264enc") return "capsfilter caps=video/x-raw,format=NV12 ! "; + if (encode == "vaapih264enc") return "capsfilter caps=video/x-raw,format=NV12 ! "; + return null; + } + + public static string? get_encode_args(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { + // H264 + if (encode == "msdkh264enc") return @" rate-control=vbr"; + if (encode == "vaapih264enc") return @" rate-control=vbr"; + if (encode == "x264enc") return @" byte-stream=1 speed-preset=ultrafast tune=zerolatency bframes=0 cabac=false dct8x8=false"; + + // VP8 + if (encode == "vaapivp8enc" || encode == "msdkvp8enc") return " rate-control=vbr target-percentage=90"; + if (encode == "vp8enc") return " deadline=1 error-resilient=3 lag-in-frames=0 resize-allowed=true threads=8 dropframe-threshold=30 end-usage=vbr cpu-used=4"; + + // VP9 + if (encode == "msdkvp9enc" || encode == "vaapivp9enc") return " rate-control=vbr target-percentage=90"; + if (encode == "vp9enc") return " deadline=1 error-resilient=3 lag-in-frames=0 resize-allowed=true threads=8 dropframe-threshold=30 end-usage=vbr cpu-used=4"; + + // OPUS + if (encode == "opusenc") { + if (payload_type != null && payload_type.parameters.has("useinbandfec", "1")) return " audio-type=voice inband-fec=true"; + return " audio-type=voice"; + } + + return null; + } + + public static string? get_encode_suffix(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { + // H264 + if (media == "video" && codec == "h264") return " ! capsfilter caps=video/x-h264,profile=constrained-baseline ! h264parse"; + if (media == "video" && codec == "vp8" && encode == "vp8enc") return " ! capsfilter caps=video/x-vp8,profile=(string)1"; + return null; + } + + public uint update_bitrate(string media, JingleRtp.PayloadType payload_type, Gst.Element encode_element, uint bitrate) { + Gst.Bin? encode_bin = encode_element as Gst.Bin; + if (encode_bin == null) return 0; + string? codec = get_codec_from_payload(media, payload_type); + string? encode_name = get_encode_element_name(media, codec); + if (encode_name == null) return 0; + Gst.Element encode = encode_bin.get_by_name(@"$(encode_bin.name)_encode"); + + switch (encode_name) { + case "msdkh264enc": + case "vaapih264enc": + case "x264enc": + case "msdkvp9enc": + case "vaapivp9enc": + case "msdkvp8enc": + case "vaapivp8enc": + bitrate = uint.min(2048000, bitrate); + encode.set("bitrate", bitrate); + return bitrate; + case "vp9enc": + case "vp8enc": + bitrate = uint.min(2147483, bitrate); + encode.set("target-bitrate", bitrate * 1024); + return bitrate; + } + + return 0; + } + + public void update_rescale_caps(Gst.Element encode_element, Gst.Caps caps) { + Gst.Bin? encode_bin = encode_element as Gst.Bin; + if (encode_bin == null) return; + Gst.Element rescale_caps = encode_bin.get_by_name(@"$(encode_bin.name)_rescale_caps"); + rescale_caps.set("caps", caps); + } + + public Gst.Caps? get_rescale_caps(Gst.Element encode_element) { + Gst.Bin? encode_bin = encode_element as Gst.Bin; + if (encode_bin == null) return null; + Gst.Element rescale_caps = encode_bin.get_by_name(@"$(encode_bin.name)_rescale_caps"); + Gst.Caps caps; + rescale_caps.get("caps", out caps); + return caps; + } + + public static string? get_decode_prefix(string media, string codec, string decode, JingleRtp.PayloadType? payload_type) { + return null; + } + + public static string? get_decode_args(string media, string codec, string decode, JingleRtp.PayloadType? payload_type) { + if (decode == "opusdec" && payload_type != null && payload_type.parameters.has("useinbandfec", "1")) return " use-inband-fec=true"; + if (decode == "vaapivp9dec" || decode == "vaapivp8dec" || decode == "vaapih264dec") return " max-errors=100"; + if (decode == "vp8dec" || decode == "vp9dec") return " threads=8"; + return null; + } + + public static string? get_decode_suffix(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { + return null; + } + + public static string? get_depay_args(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { + if (codec == "vp8") return " wait-for-keyframe=true"; + return null; + } + + public bool is_element_supported(string? element_name) { + if (element_name == null) return false; + if (unsupported_elements.contains(element_name)) return false; + if (supported_elements.contains(element_name)) return true; + var test_element = Gst.ElementFactory.make(element_name, @"test-$element_name"); + if (test_element != null) { + supported_elements.add(element_name); + return true; + } else { + warning("%s is not installed or supported on this system", element_name); + unsupported_elements.add(element_name); + return false; + } + } + + public string? get_encode_element_name(string media, string? codec) { + if (get_pay_element_name(media, codec) == null) return null; + foreach (string candidate in get_encode_candidates(media, codec)) { + if (is_element_supported(candidate)) return candidate; + } + return null; + } + + public string? get_pay_element_name(string media, string? codec) { + string? candidate = get_pay_candidate(media, codec); + if (candidate != null && is_element_supported(candidate)) return candidate; + return null; + } + + public string? get_decode_element_name(string media, string? codec) { + if (get_depay_element_name(media, codec) == null) return null; + foreach (string candidate in get_decode_candidates(media, codec)) { + if (is_element_supported(candidate)) return candidate; + } + return null; + } + + public string? get_depay_element_name(string media, string? codec) { + string? candidate = get_depay_candidate(media, codec); + if (candidate != null && is_element_supported(candidate)) return candidate; + return null; + } + + public void mark_element_unsupported(string element_name) { + unsupported_elements.add(element_name); + } + + public string? get_decode_bin_description(string media, string? codec, JingleRtp.PayloadType? payload_type, string? element_name = null, string? name = null) { + if (codec == null) return null; + string base_name = name ?? @"encode-$codec-$(Random.next_int())"; + string? depay = get_depay_element_name(media, codec); + string? decode = element_name ?? get_decode_element_name(media, codec); + if (depay == null || decode == null) return null; + string decode_prefix = get_decode_prefix(media, codec, decode, payload_type) ?? ""; + string decode_args = get_decode_args(media, codec, decode, payload_type) ?? ""; + string decode_suffix = get_decode_suffix(media, codec, decode, payload_type) ?? ""; + string depay_args = get_depay_args(media, codec, decode, payload_type) ?? ""; + string resample = media == "audio" ? @" ! audioresample name=$(base_name)_resample" : ""; + return @"queue ! $depay$depay_args name=$(base_name)_rtp_depay ! $decode_prefix$decode$decode_args name=$(base_name)_$(codec)_decode$decode_suffix ! $(media)convert name=$(base_name)_convert$resample"; + } + + public Gst.Element? get_decode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { + string? codec = get_codec_from_payload(media, payload_type); + string base_name = name ?? @"decode_$(codec)_$(Random.next_int())"; + string? desc = get_decode_bin_description(media, codec, payload_type, null, base_name); + if (desc == null) return null; + debug("Pipeline to decode %s %s: %s", media, codec, desc); + Gst.Element bin = Gst.parse_bin_from_description(desc, true); + bin.name = name; + return bin; + } + + public string? get_encode_bin_description(string media, string? codec, JingleRtp.PayloadType? payload_type, string? element_name = null, string? name = null) { + string? desc1 = get_encode_bin_without_payloader_description(media, codec, payload_type, element_name, name); + string? desc2 = get_payloader_bin_description(media, codec, payload_type, name); + return @"$desc1 ! $desc2"; + } + + public string? get_payloader_bin_description(string media, string? codec, JingleRtp.PayloadType? payload_type, string? name = null) { + if (codec == null) return null; + string base_name = name ?? @"encode_$(codec)_$(Random.next_int())"; + string? pay = get_pay_element_name(media, codec); + if (pay == null) return null; + return @"$pay pt=$(payload_type != null ? payload_type.id : 96) name=$(base_name)_rtp_pay"; + } + + public string? get_encode_bin_without_payloader_description(string media, string? codec, JingleRtp.PayloadType? payload_type, string? element_name = null, string? name = null) { + if (codec == null) return null; + string base_name = name ?? @"encode_$(codec)_$(Random.next_int())"; + string? encode = element_name ?? get_encode_element_name(media, codec); + if (encode == null) return null; + string encode_prefix = get_encode_prefix(media, codec, encode, payload_type) ?? ""; + string encode_args = get_encode_args(media, codec, encode, payload_type) ?? ""; + string encode_suffix = get_encode_suffix(media, codec, encode, payload_type) ?? ""; + string rescale = media == "audio" ? @" ! audioresample name=$(base_name)_resample" : @" ! videoscale name=$(base_name)_rescale ! capsfilter name=$(base_name)_rescale_caps"; + return @"$(media)convert name=$(base_name)_convert$rescale ! queue ! $encode_prefix$encode$encode_args name=$(base_name)_encode$encode_suffix"; + } + + public Gst.Element? get_encode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { + string? codec = get_codec_from_payload(media, payload_type); + string base_name = name ?? @"encode_$(codec)_$(Random.next_int())"; + string? desc = get_encode_bin_description(media, codec, payload_type, null, base_name); + if (desc == null) return null; + debug("Pipeline to encode %s %s: %s", media, codec, desc); + Gst.Element bin = Gst.parse_bin_from_description(desc, true); + bin.name = name; + return bin; + } + + public Gst.Element? get_encode_bin_without_payloader(string media, JingleRtp.PayloadType payload_type, string? name = null) { + string? codec = get_codec_from_payload(media, payload_type); + string base_name = name ?? @"encode_$(codec)_$(Random.next_int())"; + string? desc = get_encode_bin_without_payloader_description(media, codec, payload_type, null, base_name); + if (desc == null) return null; + debug("Pipeline to encode %s %s without payloader: %s", media, codec, desc); + Gst.Element bin = Gst.parse_bin_from_description(desc, true); + bin.name = name; + return bin; + } + + public Gst.Element? get_payloader_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { + string? codec = get_codec_from_payload(media, payload_type); + string base_name = name ?? @"encode_$(codec)_$(Random.next_int())"; + string? desc = get_payloader_bin_description(media, codec, payload_type, base_name); + if (desc == null) return null; + debug("Pipeline to payload %s %s: %s", media, codec, desc); + Gst.Element bin = Gst.parse_bin_from_description(desc, true); + bin.name = name; + return bin; + } + +} diff --git a/main/src/plugins/rtp/device.vala b/main/src/plugins/rtp/device.vala new file mode 100644 index 00000000..bdb597b6 --- /dev/null +++ b/main/src/plugins/rtp/device.vala @@ -0,0 +1,592 @@ +using Xmpp.Xep.JingleRtp; +using Gee; + +public enum Dino.Plugins.Rtp.DeviceProtocol { + OTHER, + PIPEWIRE, + V4L2, + PULSEAUDIO, + ALSA +} + +public class Dino.Plugins.Rtp.Device : MediaDevice, Object { + private const int[] common_widths = {320, 352, 416, 480, 640, 960, 1280, 1920, 2560, 3840}; + + public Plugin plugin { get; private set; } + public CodecUtil codec_util { get { return plugin.codec_util; } } + public Gst.Device device { get; private set; } + + public string id { owned get { return device_name; }} + public string display_name { owned get { return device_display_name; }} + public string? detail_name { owned get { + if (device.properties == null) return null; + if (device.properties.has_field("alsa.card_name")) return device.properties.get_string("alsa.card_name"); + if (device.properties.has_field("alsa.name")) return device.properties.get_string("alsa.name"); + if (device.properties.has_field("alsa.id")) return device.properties.get_string("alsa.id"); + if (device.properties.has_field("api.v4l2.cap.card")) return device.properties.get_string("api.v4l2.cap.card"); + return null; + }} + public string? media { owned get { + if (device.has_classes("Audio")) { + return "audio"; + } else if (device.has_classes("Video")) { + return "video"; + } else { + return null; + } + }} + public bool incoming { get { return is_sink; } } + + public Gst.Pipeline pipe { get { return plugin.pipe; }} + public bool is_source { get { return device.has_classes("Source"); }} + public bool is_sink { get { return device.has_classes("Sink"); }} + public bool is_monitor { get { return (device.properties != null && device.properties.get_string("device.class") == "monitor") || (protocol == DeviceProtocol.PIPEWIRE && device.has_classes("Stream")); } } + public bool is_default { get { + if (device.properties == null) return false; + bool ret; + device.properties.get_boolean("is-default", out ret); + return ret; + }} + public DeviceProtocol protocol { get { + if (device.properties == null) return DeviceProtocol.OTHER; + if (device.properties.has_name("pulse-proplist")) return DeviceProtocol.PULSEAUDIO; + if (device.properties.has_name("pipewire-proplist")) return DeviceProtocol.PIPEWIRE; + if (device.properties.has_name("v4l2deviceprovider")) return DeviceProtocol.V4L2; + return DeviceProtocol.OTHER; + }} + + private string device_name; + private string device_display_name; + + private Gst.Caps device_caps; + private Gst.Element element; + private Gst.Element tee; + private Gst.Element dsp; + private Gst.Base.Aggregator mixer; + private Gst.Element filter; + private int links; + + // Codecs + private Gee.Map<PayloadType, Gst.Element> codecs = new HashMap<PayloadType, Gst.Element>(PayloadType.hash_func, PayloadType.equals_func); + private Gee.Map<PayloadType, Gst.Element> codec_tees = new HashMap<PayloadType, Gst.Element>(PayloadType.hash_func, PayloadType.equals_func); + + // Payloaders + private Gee.Map<PayloadType, Gee.Map<uint, Gst.Element>> payloaders = new HashMap<PayloadType, Gee.Map<uint, Gst.Element>>(PayloadType.hash_func, PayloadType.equals_func); + private Gee.Map<PayloadType, Gee.Map<uint, Gst.Element>> payloader_tees = new HashMap<PayloadType, Gee.Map<uint, Gst.Element>>(PayloadType.hash_func, PayloadType.equals_func); + private Gee.Map<PayloadType, Gee.Map<uint, uint>> payloader_links = new HashMap<PayloadType, Gee.Map<uint, uint>>(PayloadType.hash_func, PayloadType.equals_func); + + // Bitrate + private Gee.Map<PayloadType, Gee.List<CodecBitrate>> codec_bitrates = new HashMap<PayloadType, Gee.List<CodecBitrate>>(PayloadType.hash_func, PayloadType.equals_func); + + private class CodecBitrate { + public uint bitrate; + public int64 timestamp; + + public CodecBitrate(uint bitrate) { + this.bitrate = bitrate; + this.timestamp = get_monotonic_time(); + } + } + + public Device(Plugin plugin, Gst.Device device) { + this.plugin = plugin; + update(device); + } + + public bool matches(Gst.Device device) { + if (this.device.name == device.name) return true; + return false; + } + + public void update(Gst.Device device) { + this.device = device; + this.device_name = device.name; + this.device_display_name = device.display_name; + } + + public Gst.Element? link_sink() { + if (!is_sink) return null; + if (element == null) create(); + links++; + if (mixer != null) { + Gst.Element rate = Gst.ElementFactory.make("audiorate", @"$(id)_rate_$(Random.next_int())"); + pipe.add(rate); + rate.link(mixer); + return rate; + } + if (media == "audio") return filter; + return element; + } + + public Gst.Element? link_source(PayloadType? payload_type = null, uint ssrc = 0, int seqnum_offset = -1, uint32 timestamp_offset = 0) { + if (!is_source) return null; + if (element == null) create(); + links++; + if (payload_type != null && ssrc != 0 && tee != null) { + bool new_codec = false; + string? codec = CodecUtil.get_codec_from_payload(media, payload_type); + if (!codecs.has_key(payload_type)) { + codecs[payload_type] = codec_util.get_encode_bin_without_payloader(media, payload_type, @"$(id)_$(codec)_encoder"); + pipe.add(codecs[payload_type]); + new_codec = true; + } + if (!codec_tees.has_key(payload_type)) { + codec_tees[payload_type] = Gst.ElementFactory.make("tee", @"$(id)_$(codec)_tee"); + codec_tees[payload_type].@set("allow-not-linked", true); + pipe.add(codec_tees[payload_type]); + codecs[payload_type].link(codec_tees[payload_type]); + } + if (!payloaders.has_key(payload_type)) { + payloaders[payload_type] = new HashMap<uint, Gst.Element>(); + } + if (!payloaders[payload_type].has_key(ssrc)) { + payloaders[payload_type][ssrc] = codec_util.get_payloader_bin(media, payload_type, @"$(id)_$(codec)_$(ssrc)"); + var payload = (Gst.RTP.BasePayload) ((Gst.Bin) payloaders[payload_type][ssrc]).get_by_name(@"$(id)_$(codec)_$(ssrc)_rtp_pay"); + payload.ssrc = ssrc; + payload.seqnum_offset = seqnum_offset; + if (timestamp_offset != 0) { + payload.timestamp_offset = timestamp_offset; + } + pipe.add(payloaders[payload_type][ssrc]); + codec_tees[payload_type].link(payloaders[payload_type][ssrc]); + debug("Payload for %s with %s using ssrc %u, seqnum_offset %u, timestamp_offset %u", media, codec, ssrc, seqnum_offset, timestamp_offset); + } + if (!payloader_tees.has_key(payload_type)) { + payloader_tees[payload_type] = new HashMap<uint, Gst.Element>(); + } + if (!payloader_tees[payload_type].has_key(ssrc)) { + payloader_tees[payload_type][ssrc] = Gst.ElementFactory.make("tee", @"$(id)_$(codec)_$(ssrc)_tee"); + payloader_tees[payload_type][ssrc].@set("allow-not-linked", true); + pipe.add(payloader_tees[payload_type][ssrc]); + payloaders[payload_type][ssrc].link(payloader_tees[payload_type][ssrc]); + } + if (!payloader_links.has_key(payload_type)) { + payloader_links[payload_type] = new HashMap<uint, uint>(); + } + if (!payloader_links[payload_type].has_key(ssrc)) { + payloader_links[payload_type][ssrc] = 1; + } else { + payloader_links[payload_type][ssrc] = payloader_links[payload_type][ssrc] + 1; + } + if (new_codec) { + tee.link(codecs[payload_type]); + } + return payloader_tees[payload_type][ssrc]; + } + if (tee != null) return tee; + return element; + } + + private static double get_target_bitrate(Gst.Caps caps) { + if (caps == null || caps.get_size() == 0) return uint.MAX; + unowned Gst.Structure? that = caps.get_structure(0); + int num = 0, den = 0, width = 0, height = 0; + if (!that.has_field("width") || !that.get_int("width", out width)) return uint.MAX; + if (!that.has_field("height") || !that.get_int("height", out height)) return uint.MAX; + if (!that.has_field("framerate")) return uint.MAX; + Value framerate = that.get_value("framerate"); + if (framerate.type() != typeof(Gst.Fraction)) return uint.MAX; + num = Gst.Value.get_fraction_numerator(framerate); + den = Gst.Value.get_fraction_denominator(framerate); + double pxs = ((double)num/(double)den) * (double)width * (double)height; + double br = Math.sqrt(Math.sqrt(pxs)) * 100.0 - 3700.0; + if (br < 128.0) return 128.0; + return br; + } + + private Gst.Caps get_active_caps(PayloadType payload_type) { + return codec_util.get_rescale_caps(codecs[payload_type]) ?? device_caps; + } + + private void apply_caps(PayloadType payload_type, Gst.Caps caps) { + plugin.pause(); + debug("Set scaled caps to %s", caps.to_string()); + codec_util.update_rescale_caps(codecs[payload_type], caps); + plugin.unpause(); + } + + private void apply_width(PayloadType payload_type, int new_width, uint bitrate) { + int device_caps_width, device_caps_height, active_caps_width, device_caps_framerate_num, device_caps_framerate_den; + device_caps.get_structure(0).get_int("width", out device_caps_width); + device_caps.get_structure(0).get_int("height", out device_caps_height); + device_caps.get_structure(0).get_fraction("framerate", out device_caps_framerate_num, out device_caps_framerate_den); + Gst.Caps active_caps = get_active_caps(payload_type); + if (active_caps != null && active_caps.get_size() > 0) { + active_caps.get_structure(0).get_int("width", out active_caps_width); + } else { + active_caps_width = device_caps_width; + } + if (new_width == active_caps_width) return; + int new_height = device_caps_height * new_width / device_caps_width; + if ((new_height % 2) != 0) { + new_height += 1; + } + Gst.Caps new_caps; + if (device_caps_framerate_den != 0) { + new_caps = new Gst.Caps.simple("video/x-raw", "width", typeof(int), new_width, "height", typeof(int), new_height, "framerate", typeof(Gst.Fraction), device_caps_framerate_num, device_caps_framerate_den, null); + } else { + new_caps = new Gst.Caps.simple("video/x-raw", "width", typeof(int), new_width, "height", typeof(int), new_height, null); + } + double required_bitrate = get_target_bitrate(new_caps); + debug("Changing resolution width from %d to %d (requires bitrate %f, current target is %u)", active_caps_width, new_width, required_bitrate, bitrate); + if (bitrate < required_bitrate && new_width > active_caps_width) return; + apply_caps(payload_type, new_caps); + } + + public void update_bitrate(PayloadType payload_type, uint bitrate) { + if (codecs.has_key(payload_type)) { + lock(codec_bitrates); + if (!codec_bitrates.has_key(payload_type)) { + codec_bitrates[payload_type] = new ArrayList<CodecBitrate>(); + } + codec_bitrates[payload_type].add(new CodecBitrate(bitrate)); + var remove = new ArrayList<CodecBitrate>(); + foreach (CodecBitrate rate in codec_bitrates[payload_type]) { + if (rate.timestamp < get_monotonic_time() - 5000000L) { + remove.add(rate); + continue; + } + if (rate.bitrate < bitrate) { + bitrate = rate.bitrate; + } + } + codec_bitrates[payload_type].remove_all(remove); + if (media == "video") { + if (bitrate < 128) bitrate = 128; + Gst.Caps active_caps = get_active_caps(payload_type); + double max_bitrate = get_target_bitrate(device_caps) * 2; + double current_target_bitrate = get_target_bitrate(active_caps); + int device_caps_width, active_caps_width; + device_caps.get_structure(0).get_int("width", out device_caps_width); + if (active_caps != null && active_caps.get_size() > 0) { + active_caps.get_structure(0).get_int("width", out active_caps_width); + } else { + active_caps_width = device_caps_width; + } + if (bitrate < 0.75 * current_target_bitrate && active_caps_width > common_widths[0]) { + // Lower video resolution + int i = 1; + for(; i < common_widths.length && common_widths[i] < active_caps_width; i++);if (common_widths[i] != active_caps_width) { + debug("Decrease resolution to ensure target bitrate (%u) is in reach (current resolution target bitrate is %f)", bitrate, current_target_bitrate); + } + apply_width(payload_type, common_widths[i-1], bitrate); + } else if (bitrate > 2 * current_target_bitrate && active_caps_width < device_caps_width) { + // Higher video resolution + int i = 0; + for(; i < common_widths.length && common_widths[i] <= active_caps_width; i++); + if (common_widths[i] != active_caps_width) { + debug("Increase resolution to make use of available bandwidth of target bitrate (%u) (current resolution target bitrate is %f)", bitrate, current_target_bitrate); + } + if (common_widths[i] > device_caps_width) { + // We never scale up, so just stick with what the device gives + apply_width(payload_type, device_caps_width, bitrate); + } else if (common_widths[i] != active_caps_width) { + apply_width(payload_type, common_widths[i], bitrate); + } + } + if (bitrate > max_bitrate) bitrate = (uint) max_bitrate; + } + codec_util.update_bitrate(media, payload_type, codecs[payload_type], bitrate); + unlock(codec_bitrates); + } + } + + public void unlink(Gst.Element? link = null) { + if (links <= 0) { + critical("Link count below zero."); + return; + } + if (link != null && is_source && tee != null) { + PayloadType payload_type = payloader_tees.first_match((entry) => entry.value.any_match((entry) => entry.value == link)).key; + uint ssrc = payloader_tees[payload_type].first_match((entry) => entry.value == link).key; + payloader_links[payload_type][ssrc] = payloader_links[payload_type][ssrc] - 1; + if (payloader_links[payload_type][ssrc] == 0) { + plugin.pause(); + + codec_tees[payload_type].unlink(payloaders[payload_type][ssrc]); + payloaders[payload_type][ssrc].set_locked_state(true); + payloaders[payload_type][ssrc].set_state(Gst.State.NULL); + payloaders[payload_type][ssrc].unlink(payloader_tees[payload_type][ssrc]); + pipe.remove(payloaders[payload_type][ssrc]); + payloaders[payload_type].unset(ssrc); + payloader_tees[payload_type][ssrc].set_locked_state(true); + payloader_tees[payload_type][ssrc].set_state(Gst.State.NULL); + pipe.remove(payloader_tees[payload_type][ssrc]); + payloader_tees[payload_type].unset(ssrc); + + payloader_links[payload_type].unset(ssrc); + plugin.unpause(); + } + if (payloader_links[payload_type].size == 0) { + plugin.pause(); + + tee.unlink(codecs[payload_type]); + codecs[payload_type].set_locked_state(true); + codecs[payload_type].set_state(Gst.State.NULL); + codecs[payload_type].unlink(codec_tees[payload_type]); + pipe.remove(codecs[payload_type]); + codecs.unset(payload_type); + codec_tees[payload_type].set_locked_state(true); + codec_tees[payload_type].set_state(Gst.State.NULL); + pipe.remove(codec_tees[payload_type]); + codec_tees.unset(payload_type); + + payloaders.unset(payload_type); + payloader_tees.unset(payload_type); + payloader_links.unset(payload_type); + plugin.unpause(); + } + } + if (link != null && is_sink && mixer != null) { + plugin.pause(); + link.set_locked_state(true); + Gst.Base.AggregatorPad mixer_sink_pad = (Gst.Base.AggregatorPad) link.get_static_pad("src").get_peer(); + link.get_static_pad("src").unlink(mixer_sink_pad); + mixer_sink_pad.set_active(false); + link.set_state(Gst.State.NULL); + pipe.remove(link); + mixer.release_request_pad(mixer_sink_pad); + plugin.unpause(); + } + links--; + if (links == 0) { + destroy(); + } + } + + private Gst.Caps get_best_caps() { + if (media == "audio") { + return Gst.Caps.from_string("audio/x-raw,rate=48000,channels=1"); + } else if (media == "video" && device.caps.get_size() > 0) { + int best_index = -1; + Value? best_fraction = null; + int best_fps = 0; + int best_width = 0; + int best_height = 0; + for (int i = 0; i < device.caps.get_size(); i++) { + unowned Gst.Structure? that = device.caps.get_structure(i); + unowned Gst.CapsFeatures? features = device.caps.get_features(i); + Value? best_fraction_now = null; + if (!that.has_name("video/x-raw") || !features.contains("memory:SystemMemory")) continue; + int num = 0, den = 0, width = 0, height = 0; + if (!that.has_field("framerate")) continue; + Value framerate = that.get_value("framerate"); + if (framerate.type() == typeof(Gst.Fraction)) { + num = Gst.Value.get_fraction_numerator(framerate); + den = Gst.Value.get_fraction_denominator(framerate); + } else if (framerate.type() == typeof(Gst.ValueList)) { + for(uint j = 0; j < Gst.ValueList.get_size(framerate); j++) { + Value fraction = Gst.ValueList.get_value(framerate, j); + int in_num = Gst.Value.get_fraction_numerator(fraction); + int in_den = Gst.Value.get_fraction_denominator(fraction); + int fps = den > 0 ? (num/den) : 0; + int in_fps = in_den > 0 ? (in_num/in_den) : 0; + if (in_fps > fps) { + best_fraction_now = fraction; + num = in_num; + den = in_den; + } + } + } else { + debug("Unknown type for framerate: %s", framerate.type_name()); + } + if (den == 0) continue; + if (!that.has_field("width") || !that.get_int("width", out width)) continue; + if (!that.has_field("height") || !that.get_int("height", out height)) continue; + int fps = num/den; + if (best_fps < fps || best_fps == fps && best_width < width || best_fps == fps && best_width == width && best_height < height) { + best_fps = fps; + best_width = width; + best_height = height; + best_index = i; + best_fraction = best_fraction_now; + } + } + if (best_index == -1) { + // No caps in first round, try without framerate + int target_pixel_count = 1280 * 720; + double target_pixel_ratio = 16.0 / 9.0; + double best_diversion_factor = 0.0; + + for (int i = 0; i < device.caps.get_size(); i++) { + unowned Gst.Structure? that = device.caps.get_structure(i); + unowned Gst.CapsFeatures? features = device.caps.get_features(i); + if (!that.has_name("video/x-raw") || !features.contains("memory:SystemMemory")) continue; + int width = 0, height = 0; + if (!that.has_field("width") || !that.get_int("width", out width) || width == 0) continue; + if (!that.has_field("height") || !that.get_int("height", out height) || height == 0) continue; + + int pixel_count = width * height; + double pixel_count_diversion_factor; + if (pixel_count > target_pixel_count) + pixel_count_diversion_factor = (double)pixel_count / target_pixel_count; + else + pixel_count_diversion_factor = (double)target_pixel_count / pixel_count; + + double pixel_ratio; + if (width > height) + pixel_ratio = (double)width / height; + else + pixel_ratio = (double)height / width; + + double ratio_diversion_factor; + if (pixel_ratio > target_pixel_ratio) + ratio_diversion_factor = pixel_ratio / target_pixel_ratio; + else + ratio_diversion_factor = target_pixel_ratio / pixel_ratio; + + // 1.0 is the best possible value. Any diversion from + // target_pixel_count or target_pixel_ratio increases it. + double diversion_factor = pixel_count_diversion_factor * ratio_diversion_factor; + + if (best_index == -1 || diversion_factor < best_diversion_factor) { + best_index = i; + best_width = width; + best_height = height; + best_diversion_factor = diversion_factor; + } + } + } + Gst.Caps res = caps_copy_nth(device.caps, best_index); + unowned Gst.Structure? that = res.get_structure(0); + Value? framerate = that.get_value("framerate"); + if (framerate != null && framerate.type() == typeof(Gst.ValueList) && best_fraction != null) { + that.set_value("framerate", best_fraction); + } else if (framerate == null) { + // The source does not support framerate negotiation. Just set a + // reasonable value. The source might be able to honor it and + // should ignore it otherwise. + that.@set("framerate", typeof(Gst.Fraction), 30, 1, null); + } + debug("Selected caps %s", res.to_string()); + return res; + } else if (device.caps.get_size() > 0) { + return caps_copy_nth(device.caps, 0); + } else { + return new Gst.Caps.any(); + } + } + + // Backport from gst_caps_copy_nth added in GStreamer 1.16 + private static Gst.Caps caps_copy_nth(Gst.Caps source, uint index) { + Gst.Caps target = new Gst.Caps.empty(); + target.flags = source.flags; + target.append_structure_full(source.get_structure(index).copy(), source.get_features(index).copy()); + return target; + } + + private void create() { + debug("Creating device %s", id); + plugin.pause(); + element = device.create_element(id); + if (is_sink) { + element.@set("async", false); + element.@set("sync", false); + } + pipe.add(element); + device_caps = get_best_caps(); + if (is_source) { + element.@set("do-timestamp", true); + filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id"); + filter.@set("caps", device_caps); + pipe.add(filter); + element.link(filter); +#if WITH_VOICE_PROCESSOR + if (media == "audio" && plugin.echoprobe != null) { + dsp = new VoiceProcessor(plugin.echoprobe as EchoProbe, element as Gst.Audio.StreamVolume); + dsp.name = @"dsp_$id"; + pipe.add(dsp); + filter.link(dsp); + } +#endif + tee = Gst.ElementFactory.make("tee", @"tee_$id"); + tee.@set("allow-not-linked", true); + pipe.add(tee); + (dsp ?? filter).link(tee); + } + if (is_sink && media == "audio") { + mixer = (Gst.Base.Aggregator) Gst.ElementFactory.make("audiomixer", @"mixer_$id"); + pipe.add(mixer); + if (plugin.echoprobe != null && !plugin.echoprobe.get_static_pad("src").is_linked()) { + mixer.link(plugin.echoprobe); + plugin.echoprobe.link(element); + } else { + filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id"); + filter.@set("caps", device_caps); + pipe.add(filter); + mixer.link(filter); + filter.link(element); + } + } + plugin.unpause(); + } + + private void destroy() { + if (is_sink) { + if (mixer != null) { + int linked_sink_pads = 0; + mixer.foreach_sink_pad((_, pad) => { + if (pad.is_linked()) linked_sink_pads++; + return true; + }); + if (linked_sink_pads > 0) { + warning("%s-mixer still has %i sink pads while being destroyed", id, linked_sink_pads); + } + mixer.unlink(plugin.echoprobe ?? element); + } + if (filter != null) { + filter.set_locked_state(true); + filter.set_state(Gst.State.NULL); + filter.unlink(element); + pipe.remove(filter); + filter = null; + } + if (plugin.echoprobe != null) { + plugin.echoprobe.unlink(element); + } + } + element.set_locked_state(true); + element.set_state(Gst.State.NULL); + if (filter != null) element.unlink(filter); + else if (is_source) element.unlink(tee); + pipe.remove(element); + element = null; + if (mixer != null) { + mixer.set_locked_state(true); + mixer.set_state(Gst.State.NULL); + pipe.remove(mixer); + mixer = null; + } + if (is_source) { + if (filter != null) { + filter.set_locked_state(true); + filter.set_state(Gst.State.NULL); + filter.unlink(dsp ?? tee); + pipe.remove(filter); + filter = null; + } + if (dsp != null) { + dsp.set_locked_state(true); + dsp.set_state(Gst.State.NULL); + dsp.unlink(tee); + pipe.remove(dsp); + dsp = null; + } + if (tee != null) { + int linked_src_pads = 0; + tee.foreach_src_pad((_, pad) => { + if (pad.is_linked()) linked_src_pads++; + return true; + }); + if (linked_src_pads != 0) { + warning("%s-tee still has %d src pads while being destroyed", id, linked_src_pads); + } + tee.set_locked_state(true); + tee.set_state(Gst.State.NULL); + pipe.remove(tee); + tee = null; + } + } + debug("Destroyed device %s", id); + } +} diff --git a/main/src/plugins/rtp/gst_fixes.c b/main/src/plugins/rtp/gst_fixes.c new file mode 100644 index 00000000..86b2b598 --- /dev/null +++ b/main/src/plugins/rtp/gst_fixes.c @@ -0,0 +1,10 @@ +#include <gst/video/video.h> + +GstVideoInfo *gst_video_frame_get_video_info(GstVideoFrame *frame) { + return &frame->info; +} + +void *gst_video_frame_get_data(GstVideoFrame *frame, size_t* length) { + *length = frame->info.height * frame->info.stride[0]; + return frame->data[0]; +}
\ No newline at end of file diff --git a/main/src/plugins/rtp/module.vala b/main/src/plugins/rtp/module.vala new file mode 100644 index 00000000..be153b10 --- /dev/null +++ b/main/src/plugins/rtp/module.vala @@ -0,0 +1,235 @@ +using Gee; +using Xmpp; +using Xmpp.Xep; + +public class Dino.Plugins.Rtp.Module : JingleRtp.Module { + private Set<string> supported_codecs = new HashSet<string>(); + private Set<string> unsupported_codecs = new HashSet<string>(); + public Plugin plugin { get; private set; } + public CodecUtil codec_util { get { + return plugin.codec_util; + }} + + public Module(Plugin plugin) { + base(); + this.plugin = plugin; + } + + private async bool pipeline_works(string media, string element_desc) { + var supported = false; + string pipeline_desc = @"$(media)testsrc is-live=true ! $element_desc ! appsink name=output"; + try { + var pipeline = Gst.parse_launch(pipeline_desc); + var output = ((Gst.Bin) pipeline).get_by_name("output") as Gst.App.Sink; + SourceFunc callback = pipeline_works.callback; + var finished = false; + output.emit_signals = true; + output.new_sample.connect(() => { + if (!finished) { + finished = true; + supported = true; + Idle.add(() => { + callback(); + return Source.REMOVE; + }); + } + return Gst.FlowReturn.EOS; + }); + pipeline.bus.add_watch(Priority.DEFAULT, (_, message) => { + if (message.type == Gst.MessageType.ERROR && !finished) { + Error e; + string d; + message.parse_error(out e, out d); + debug("pipeline [%s] failed: %s", pipeline_desc, e.message); + debug(d); + finished = true; + callback(); + } + return true; + }); + Timeout.add(2000, () => { + if (!finished) { + finished = true; + callback(); + } + return Source.REMOVE; + }); + pipeline.set_state(Gst.State.PLAYING); + yield; + pipeline.set_state(Gst.State.NULL); + } catch (Error e) { + debug("pipeline [%s] failed: %s", pipeline_desc, e.message); + } + return supported; + } + + public override async bool is_payload_supported(string media, JingleRtp.PayloadType payload_type) { + string? codec = CodecUtil.get_codec_from_payload(media, payload_type); + if (codec == null) return false; + if (unsupported_codecs.contains(codec)) return false; + if (supported_codecs.contains(codec)) return true; + + string? encode_element = codec_util.get_encode_element_name(media, codec); + string? decode_element = codec_util.get_decode_element_name(media, codec); + if (encode_element == null || decode_element == null) { + warning("No suitable encoder or decoder found for %s", codec); + unsupported_codecs.add(codec); + return false; + } + + string encode_bin = codec_util.get_encode_bin_description(media, codec, null, encode_element); + while (!(yield pipeline_works(media, encode_bin))) { + debug("%s not suited for encoding %s", encode_element, codec); + codec_util.mark_element_unsupported(encode_element); + encode_element = codec_util.get_encode_element_name(media, codec); + if (encode_element == null) { + warning("No suitable encoder found for %s", codec); + unsupported_codecs.add(codec); + return false; + } + encode_bin = codec_util.get_encode_bin_description(media, codec, null, encode_element); + } + debug("using %s to encode %s", encode_element, codec); + + string decode_bin = codec_util.get_decode_bin_description(media, codec, null, decode_element); + while (!(yield pipeline_works(media, @"$encode_bin ! $decode_bin"))) { + debug("%s not suited for decoding %s", decode_element, codec); + codec_util.mark_element_unsupported(decode_element); + decode_element = codec_util.get_decode_element_name(media, codec); + if (decode_element == null) { + warning("No suitable decoder found for %s", codec); + unsupported_codecs.add(codec); + return false; + } + decode_bin = codec_util.get_decode_bin_description(media, codec, null, decode_element); + } + debug("using %s to decode %s", decode_element, codec); + + supported_codecs.add(codec); + return true; + } + + public override bool is_header_extension_supported(string media, JingleRtp.HeaderExtension ext) { + if (media == "video" && ext.uri == "urn:3gpp:video-orientation") return true; + return false; + } + + public override Gee.List<JingleRtp.HeaderExtension> get_suggested_header_extensions(string media) { + Gee.List<JingleRtp.HeaderExtension> exts = new ArrayList<JingleRtp.HeaderExtension>(); + if (media == "video") { + exts.add(new JingleRtp.HeaderExtension(1, "urn:3gpp:video-orientation")); + } + return exts; + } + + public async void add_if_supported(Gee.List<JingleRtp.PayloadType> list, string media, JingleRtp.PayloadType payload_type) { + if (yield is_payload_supported(media, payload_type)) { + list.add(payload_type); + } + } + + public override async Gee.List<JingleRtp.PayloadType> get_supported_payloads(string media) { + Gee.List<JingleRtp.PayloadType> list = new ArrayList<JingleRtp.PayloadType>(JingleRtp.PayloadType.equals_func); + if (media == "audio") { + var opus = new JingleRtp.PayloadType() { channels = 1, clockrate = 48000, name = "opus", id = 111, channels = 2 }; + opus.parameters["useinbandfec"] = "1"; + var speex32 = new JingleRtp.PayloadType() { channels = 1, clockrate = 32000, name = "speex", id = 112 }; + var speex16 = new JingleRtp.PayloadType() { channels = 1, clockrate = 16000, name = "speex", id = 113 }; + var speex8 = new JingleRtp.PayloadType() { channels = 1, clockrate = 8000, name = "speex", id = 114 }; + var g722 = new JingleRtp.PayloadType() { channels = 1, clockrate = 8000, name = "G722", id = 9 }; + var pcmu = new JingleRtp.PayloadType() { channels = 1, clockrate = 8000, name = "PCMU", id = 0 }; + var pcma = new JingleRtp.PayloadType() { channels = 1, clockrate = 8000, name = "PCMA", id = 8 }; + yield add_if_supported(list, media, opus); + yield add_if_supported(list, media, speex32); + yield add_if_supported(list, media, speex16); + yield add_if_supported(list, media, speex8); + yield add_if_supported(list, media, g722); + yield add_if_supported(list, media, pcmu); + yield add_if_supported(list, media, pcma); + } else if (media == "video") { + var rtcp_fbs = new ArrayList<JingleRtp.RtcpFeedback>(); + rtcp_fbs.add(new JingleRtp.RtcpFeedback("goog-remb")); + rtcp_fbs.add(new JingleRtp.RtcpFeedback("ccm", "fir")); + rtcp_fbs.add(new JingleRtp.RtcpFeedback("nack")); + rtcp_fbs.add(new JingleRtp.RtcpFeedback("nack", "pli")); +#if ENABLE_H264 + var h264 = new JingleRtp.PayloadType() { clockrate = 90000, name = "H264", id = 96 }; + yield add_if_supported(list, media, h264); + h264.rtcp_fbs.add_all(rtcp_fbs); +#endif +#if ENABLE_VP9 + var vp9 = new JingleRtp.PayloadType() { clockrate = 90000, name = "VP9", id = 97 }; + vp9.rtcp_fbs.add_all(rtcp_fbs); + yield add_if_supported(list, media, vp9); +#endif + var vp8 = new JingleRtp.PayloadType() { clockrate = 90000, name = "VP8", id = 98 }; + vp8.rtcp_fbs.add_all(rtcp_fbs); + yield add_if_supported(list, media, vp8); + } else { + warning("Unsupported media type: %s", media); + } + return list; + } + + public override async JingleRtp.PayloadType? pick_payload_type(string media, Gee.List<JingleRtp.PayloadType> payloads) { + if (media == "audio" || media == "video") { + foreach (JingleRtp.PayloadType type in payloads) { + if (yield is_payload_supported(media, type)) return adjust_payload_type(media, type.clone()); + } + } else { + warning("Unsupported media type: %s", media); + } + return null; + } + + public JingleRtp.PayloadType adjust_payload_type(string media, JingleRtp.PayloadType type) { + var iter = type.rtcp_fbs.iterator(); + while (iter.next()) { + var fb = iter.@get(); + switch (fb.type_) { + case "goog-remb": + if (fb.subtype != null) iter.remove(); + break; + case "ccm": + if (fb.subtype != "fir") iter.remove(); + break; + case "nack": + if (fb.subtype != null && fb.subtype != "pli") iter.remove(); + break; + default: + iter.remove(); + break; + } + } + return type; + } + + public override JingleRtp.Stream create_stream(Jingle.Content content) { + return plugin.open_stream(content); + } + + public override void close_stream(JingleRtp.Stream stream) { + var rtp_stream = stream as Rtp.Stream; + plugin.close_stream(rtp_stream); + } + + public override JingleRtp.Crypto? generate_local_crypto() { + uint8[] key_and_salt = new uint8[30]; + Crypto.randomize(key_and_salt); + return JingleRtp.Crypto.create(JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_80, key_and_salt); + } + + public override JingleRtp.Crypto? pick_remote_crypto(Gee.List<JingleRtp.Crypto> cryptos) { + foreach (JingleRtp.Crypto crypto in cryptos) { + if (crypto.is_valid) return crypto; + } + return null; + } + + public override JingleRtp.Crypto? pick_local_crypto(JingleRtp.Crypto? remote) { + if (remote == null || !remote.is_valid) return null; + uint8[] key_and_salt = new uint8[30]; + Crypto.randomize(key_and_salt); + return remote.rekey(key_and_salt); + } +} diff --git a/main/src/plugins/rtp/participant.vala b/main/src/plugins/rtp/participant.vala new file mode 100644 index 00000000..1ca13191 --- /dev/null +++ b/main/src/plugins/rtp/participant.vala @@ -0,0 +1,39 @@ +using Gee; +using Xmpp; + +public class Dino.Plugins.Rtp.Participant { + public Jid full_jid { get; private set; } + + protected Gst.Pipeline pipe; + private Map<Stream, uint32> ssrcs = new HashMap<Stream, uint32>(); + + public Participant(Gst.Pipeline pipe, Jid full_jid) { + this.pipe = pipe; + this.full_jid = full_jid; + } + + public uint32 get_ssrc(Stream stream) { + if (ssrcs.has_key(stream)) { + return ssrcs[stream]; + } + return 0; + } + + public void set_ssrc(Stream stream, uint32 ssrc) { + if (ssrcs.has_key(stream)) { + warning("Learning ssrc %ul for %s in %s when it is already known as %ul", ssrc, full_jid.to_string(), stream.to_string(), ssrcs[stream]); + } else { + stream.on_destroy.connect(unset_ssrc); + } + ssrcs[stream] = ssrc; + } + + public void unset_ssrc(Stream stream) { + ssrcs.unset(stream); + stream.on_destroy.disconnect(unset_ssrc); + } + + public string to_string() { + return @"participant $full_jid"; + } +}
\ No newline at end of file diff --git a/main/src/plugins/rtp/plugin.vala b/main/src/plugins/rtp/plugin.vala new file mode 100644 index 00000000..d09b7edf --- /dev/null +++ b/main/src/plugins/rtp/plugin.vala @@ -0,0 +1,504 @@ +using Gee; +using Xmpp; +using Xmpp.Xep; + +public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { + public Dino.Application app { get; private set; } + public CodecUtil codec_util { get; private set; } + public Gst.DeviceMonitor? device_monitor { get; private set; } + public Gst.Pipeline? pipe { get; private set; } + public Gst.Bin? rtpbin { get; private set; } + public Gst.Element? echoprobe { get; private set; } + + private Gee.List<Stream> streams = new ArrayList<Stream>(); + private Gee.List<Device> devices = new ArrayList<Device>(); + private int64 last_devices_refresh = 0; + // private Gee.List<Participant> participants = new ArrayList<Participant>(); + + public void registered(Dino.Application app) { + this.app = app; + this.codec_util = new CodecUtil(); + app.add_option_group(Gst.init_get_option_group()); + app.stream_interactor.module_manager.initialize_account_modules.connect((account, list) => { + list.add(new Module(this)); + }); + app.plugin_registry.video_call_plugin = this; + } + + private int pause_count = 0; + public void pause() { +// if (pause_count == 0) { +// debug("Pausing pipe for modifications"); +// pipe.set_state(Gst.State.PAUSED); +// } + pause_count++; + } + public void unpause() { + pause_count--; + if (pause_count == 0) { + debug("Continue pipe after modifications"); + pipe.set_state(Gst.State.PLAYING); + } + if (pause_count < 0) warning("Pause count below zero!"); + } + + private void handle_existing_devices(Gst.DeviceMonitor device_monitor) { + var new_devices = new ArrayList<Device>(); + foreach (Gst.Device device in device_monitor.get_devices()) { + if (device.properties == null) continue; + if (device.properties.has_name("pipewire-proplist") && device.has_classes("Audio")) continue; + if (device.properties.get_string("device.class") == "monitor") continue; + var pre_device = devices.first_match((it) => it.matches(device)); + if (pre_device != null) { + if (!new_devices.contains(pre_device)) new_devices.add(pre_device); + continue; + } + var new_device = new Device(this, device); + devices.add(new_device); + new_devices.add(new_device); + } + devices.retain_all(new_devices); + } + + private void refresh_devices() { + if (device_monitor != null) return; // No manual refresh needed, we're monitoring actively. + if (last_devices_refresh > 0 && last_devices_refresh + 5000000 > get_monotonic_time()) return; // Already current + var device_monitor = new Gst.DeviceMonitor(); + device_monitor.show_all = true; + device_monitor.start(); + handle_existing_devices(device_monitor); + device_monitor.stop(); + last_devices_refresh = get_monotonic_time(); + } + + private void start_device_monitor() { + if (device_monitor != null) return; + device_monitor = new Gst.DeviceMonitor(); + device_monitor.show_all = true; + device_monitor.get_bus().add_watch(Priority.DEFAULT, on_device_monitor_message); + device_monitor.start(); + handle_existing_devices(device_monitor); + } + + private void stop_device_monitor() { + if (device_monitor == null) return; + device_monitor.stop(); + device_monitor = null; + } + + private void init_call_pipe() { + if (pipe != null) return; + debug("Creating call pipe."); + start_device_monitor(); + pipe = new Gst.Pipeline(null); + + // RTP + rtpbin = Gst.ElementFactory.make("rtpbin", null) as Gst.Bin; + if (rtpbin == null) { + warning("RTP not supported"); + pipe = null; + return; + } + rtpbin.pad_added.connect(on_rtp_pad_added); + rtpbin.@set("latency", 100); + rtpbin.@set("do-lost", true); +// rtpbin.@set("do-sync-event", true); + rtpbin.@set("drop-on-latency", true); + rtpbin.connect("signal::request-pt-map", request_pt_map, this); + pipe.add(rtpbin); + +#if WITH_VOICE_PROCESSOR + // Audio echo probe + echoprobe = new EchoProbe(); + if (echoprobe != null) pipe.add(echoprobe); +#endif + + // Pipeline + pipe.auto_flush_bus = true; + pipe.bus.add_watch(GLib.Priority.DEFAULT, (_, message) => { + on_pipe_bus_message(message); + return true; + }); + pipe.set_state(Gst.State.PLAYING); + } + + public void destroy_call_pipe_if_unused() { + if (streams.is_empty && !VideoWidget.has_instances()) { + destroy_call_pipe(); + } + } + + private void destroy_call_pipe() { + if (pipe == null) return; + pipe.set_state(Gst.State.NULL); + rtpbin = null; +#if WITH_VOICE_PROCESSOR + echoprobe = null; +#endif + pipe = null; + stop_device_monitor(); + debug("Call pipe destroyed"); + } + + private static Gst.Caps? request_pt_map(Gst.Element rtpbin, uint session, uint pt, Plugin plugin) { + debug("request-pt-map"); + return null; + } + + private void on_rtp_pad_added(Gst.Pad pad) { + debug("pad added: %s", pad.name); + if (pad.name.has_prefix("recv_rtp_src_")) { + string[] split = pad.name.split("_"); + uint8 rtpid = (uint8)int.parse(split[3]); + foreach (Stream stream in streams) { + if (stream.rtpid == rtpid) { + stream.on_ssrc_pad_added((uint32) split[4].to_uint64(), pad); + } + } + } + if (pad.name.has_prefix("send_rtp_src_")) { + string[] split = pad.name.split("_"); + uint8 rtpid = (uint8)int.parse(split[3]); + debug("pad %s for stream %hhu", pad.name, rtpid); + foreach (Stream stream in streams) { + if (stream.rtpid == rtpid) { + stream.on_send_rtp_src_added(pad); + } + } + } + } + + private void on_pipe_bus_message(Gst.Message message) { + switch (message.type) { + case Gst.MessageType.ERROR: + Error error; + string str; + message.parse_error(out error, out str); + warning("Error in pipeline: %s", error.message); + debug(str); + break; + case Gst.MessageType.WARNING: + Error error; + string str; + message.parse_warning(out error, out str); + warning("Warning in pipeline: %s", error.message); + debug(str); + break; + case Gst.MessageType.CLOCK_LOST: + debug("Clock lost. Restarting"); + pipe.set_state(Gst.State.READY); + pipe.set_state(Gst.State.PLAYING); + break; + case Gst.MessageType.STATE_CHANGED: + // Ignore + break; + case Gst.MessageType.STREAM_STATUS: + Gst.StreamStatusType status; + Gst.Element owner; + message.parse_stream_status(out status, out owner); + if (owner != null) { + debug("%s stream changed status to %s", owner.name, status.to_string()); + } + break; + case Gst.MessageType.ELEMENT: + unowned Gst.Structure struc = message.get_structure(); + if (struc != null && message.src is Gst.Element) { + debug("Message from %s in pipeline: %s", ((Gst.Element)message.src).name, struc.to_string()); + } + break; + case Gst.MessageType.NEW_CLOCK: + debug("New clock."); + break; + case Gst.MessageType.TAG: + // Ignore + break; + case Gst.MessageType.QOS: + // Ignore + break; + case Gst.MessageType.LATENCY: + if (message.src != null && message.src.name != null && message.src is Gst.Element) { + Gst.Query latency_query = new Gst.Query.latency(); + if (((Gst.Element)message.src).query(latency_query)) { + bool live; + Gst.ClockTime min_latency, max_latency; + latency_query.parse_latency(out live, out min_latency, out max_latency); + debug("Latency message from %s: live=%s, min_latency=%s, max_latency=%s", message.src.name, live.to_string(), min_latency.to_string(), max_latency.to_string()); + } + } + break; + default: + debug("Pipe bus message: %s", message.type.to_string()); + break; + } + } + + private bool on_device_monitor_message(Gst.Bus bus, Gst.Message message) { + Gst.Device? old_gst_device = null; + Gst.Device? gst_device = null; + Device? device = null; + switch (message.type) { + case Gst.MessageType.DEVICE_ADDED: + message.parse_device_added(out gst_device); + if (devices.any_match((it) => it.matches(gst_device))) return Source.CONTINUE; + device = new Device(this, gst_device); + devices.add(device); + break; + case Gst.MessageType.DEVICE_CHANGED: + message.parse_device_changed(out gst_device, out old_gst_device); + device = devices.first_match((it) => it.matches(old_gst_device)); + if (device != null) device.update(gst_device); + break; + case Gst.MessageType.DEVICE_REMOVED: + message.parse_device_removed(out gst_device); + device = devices.first_match((it) => it.matches(gst_device)); + if (device != null) devices.remove(device); + break; + default: + break; + } + if (device != null) { + devices_changed(device.media, device.is_sink); + } + return Source.CONTINUE; + } + + public uint8 next_free_id() { + uint8 rtpid = 0; + while (streams.size < 100 && streams.any_match((stream) => stream.rtpid == rtpid)) { + rtpid++; + } + return rtpid; + } + + // public Participant get_participant(Jid full_jid, bool self) { +// foreach (Participant participant in participants) { +// if (participant.full_jid.equals(full_jid)) { +// return participant; +// } +// } +// Participant participant; +// if (self) { +// participant = new SelfParticipant(pipe, full_jid); +// } else { +// participant = new Participant(pipe, full_jid); +// } +// participants.add(participant); +// return participant; +// } + + public Stream open_stream(Xmpp.Xep.Jingle.Content content) { + init_call_pipe(); + var content_params = content.content_params as Xmpp.Xep.JingleRtp.Parameters; + if (content_params == null) return null; + Stream stream; + if (content_params.media == "video") { + stream = new VideoStream(this, content); + } else { + stream = new Stream(this, content); + } + streams.add(stream); + return stream; + } + + public void close_stream(Stream stream) { + streams.remove(stream); + stream.destroy(); + destroy_call_pipe_if_unused(); + } + + public void shutdown() { + stop_device_monitor(); + destroy_call_pipe(); + Gst.deinit(); + } + + public bool supported() { + return codec_util.is_element_supported("rtpbin"); + } + + public VideoCallWidget? create_widget(WidgetType type) { + init_call_pipe(); + if (type == WidgetType.GTK4) { + return new VideoWidget(this); + } + return null; + } + + public Gee.List<MediaDevice> get_devices(string media, bool incoming) { + refresh_devices(); + Gee.List<MediaDevice> devices; + if (media == "video" && !incoming) { + devices = get_video_sources(); + } else if (media == "audio") { + devices = get_audio_devices(incoming); + } else { + devices = new ArrayList<MediaDevice>(); + devices.add_all_iterator(this.devices.filter(it => it.media == media && (incoming && it.is_sink || !incoming && it.is_source) && !it.is_monitor)); + } + devices.sort((media_left, media_right) => { + return strcmp(media_left.id, media_right.id); + }); + + return devices; + } + + public Gee.List<MediaDevice> get_audio_devices(bool incoming) { + ArrayList<MediaDevice> pulse_devices = new ArrayList<MediaDevice>(); + ArrayList<MediaDevice> other_devices = new ArrayList<MediaDevice>(); + + foreach (Device device in devices) { + if (device.media != "audio") continue; + if (incoming && !device.is_sink || !incoming && !device.is_source) continue; + + // Skip monitors + if (device.is_monitor) continue; + + if (device.protocol == DeviceProtocol.PULSEAUDIO) { + pulse_devices.add(device); + } else { + other_devices.add(device); + } + } + + // If we have any pulseaudio devices, present only those. Don't want duplicated devices from pipewire and pulseaudio. + return pulse_devices.size > 0 ? pulse_devices : other_devices; + } + + public Gee.List<MediaDevice> get_video_sources() { + ArrayList<MediaDevice> pipewire_devices = new ArrayList<MediaDevice>(); + ArrayList<MediaDevice> other_devices = new ArrayList<MediaDevice>(); + + foreach (Device device in devices) { + if (device.media != "video") continue; + if (device.is_sink) continue; + + // Skip monitors + if (device.is_monitor) continue; + + bool is_color = false; + for (int i = 0; i < device.device.caps.get_size(); i++) { + unowned Gst.Structure structure = device.device.caps.get_structure(i); + if (!structure.has_field("format")) continue; + // "format" might be an array and get_string() will then return null. We just assume arrays to be fine. + string? format = structure.get_string("format"); + if (format == null || !format.has_prefix("GRAY")) { + is_color = true; + } + } + + // Don't allow grey-scale devices + if (!is_color) continue; + + if (device.protocol == DeviceProtocol.PIPEWIRE) { + pipewire_devices.add(device); + } else { + other_devices.add(device); + } + } + + // If we have any pipewire devices, present only those. Don't want duplicated devices from pipewire and video for linux. + return pipewire_devices.size > 0 ? pipewire_devices : other_devices; + } + + private int get_max_fps(Device device) { + int fps = 0; + for (int i = 0; i < device.device.caps.get_size(); i++) { + unowned Gst.Structure structure = device.device.caps.get_structure(i); + + if (structure.has_field("framerate")) { + Value framerate = structure.get_value("framerate"); + if (framerate.type() == typeof(Gst.Fraction)) { + int num = Gst.Value.get_fraction_numerator(framerate); + int den = Gst.Value.get_fraction_denominator(framerate); + fps = int.max(fps, num / den); + } else if (framerate.type() == typeof(Gst.ValueList)) { + for(uint j = 0; j < Gst.ValueList.get_size(framerate); j++) { + Value fraction = Gst.ValueList.get_value(framerate, j); + int num = Gst.Value.get_fraction_numerator(fraction); + int den = Gst.Value.get_fraction_denominator(fraction); + fps = int.max(fps, num / den); + } + } else { + debug("Unknown type for framerate %s on device %s", framerate.type_name(), device.display_name); + } + } + } + + debug("Max framerate for device %s: %d", device.display_name, fps); + return fps; + } + + public MediaDevice? get_preferred_device(string media, bool incoming) { + Gee.List<Device> devices = new ArrayList<Device>(); + foreach (MediaDevice media_device in get_devices(media, incoming)) { + if (media_device is Device) devices.add((Device)media_device); + } + if (devices.is_empty) { + warning("No preferred device for %s %s. Media will not be processed.", incoming ? "incoming" : "outgoing", media); + return null; + } + + // Take default if present + foreach (Device device in devices) { + if (device.is_default) { + debug("Using %s for %s %s as it's default", device.display_name, incoming ? "incoming" : "outgoing", media); + return device; + } + } + + if (media == "video") { + // Pick best FPS + int max_fps = -1; + Device? max_fps_device = null; + foreach (Device device in devices) { + int fps = get_max_fps(device); + if (fps > max_fps || max_fps_device == null) { + max_fps = fps; + max_fps_device = device; + } + } + debug("Using %s for %s %s as it has max FPS (%d)", max_fps_device.display_name, incoming ? "incoming" : "outgoing", media, max_fps); + return max_fps_device; + } else { + // Pick any + Device? device = devices.first(); + debug("Using %s for %s %s as it's first pick", device.display_name, incoming ? "incoming" : "outgoing", media); + return device; + } + } + + public MediaDevice? get_device(Xmpp.Xep.JingleRtp.Stream? stream, bool incoming) { + Stream? plugin_stream = stream as Stream?; + if (plugin_stream == null) return null; + MediaDevice? device = incoming ? plugin_stream.output_device : plugin_stream.input_device; + return device ?? get_preferred_device(stream.media, incoming); + } + + public void dump_dot() { + if (pipe == null) return; + string name = @"pipe-$(pipe.clock.get_time())-$(pipe.current_state)"; + Gst.Debug.bin_to_dot_file(pipe, Gst.DebugGraphDetails.ALL, name); + print(@"Stored pipe details as $name\n"); + } + + public void set_pause(Xmpp.Xep.JingleRtp.Stream? stream, bool pause) { + Stream? plugin_stream = stream as Stream?; + if (plugin_stream == null) return; + if (pause) { + plugin_stream.pause(); + } else { + plugin_stream.unpause(); + } + } + + public void set_device(Xmpp.Xep.JingleRtp.Stream? stream, MediaDevice? device) { + Device? real_device = device as Device?; + Stream? plugin_stream = stream as Stream?; + if (real_device == null || plugin_stream == null) return; + if (real_device.is_source) { + plugin_stream.input_device = real_device; + } else if (real_device.is_sink) { + plugin_stream.output_device = real_device; + } + } +} diff --git a/main/src/plugins/rtp/stream.vala b/main/src/plugins/rtp/stream.vala new file mode 100644 index 00000000..9a46d9ff --- /dev/null +++ b/main/src/plugins/rtp/stream.vala @@ -0,0 +1,851 @@ +using Gee; +using Xmpp; + +public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { + public uint8 rtpid { get; private set; } + + public Plugin plugin { get; private set; } + public Gst.Pipeline pipe { get { + return plugin.pipe; + }} + public Gst.Element rtpbin { get { + return plugin.rtpbin; + }} + public CodecUtil codec_util { get { + return plugin.codec_util; + }} + private Gst.App.Sink send_rtp; + private Gst.App.Sink send_rtcp; + private Gst.App.Src recv_rtp; + private Gst.App.Src recv_rtcp; + private Gst.Element decode; + private Gst.RTP.BaseDepayload decode_depay; + private Gst.Element input; + private Gst.Pad input_pad; + private Gst.Element output; + private Gst.Element session; + + private Device _input_device; + public Device input_device { get { return _input_device; } set { + if (sending && !paused) { + var input = this.input; + set_input(value != null ? value.link_source(payload_type, our_ssrc, next_seqnum_offset, next_timestamp_offset) : null); + if (this._input_device != null) this._input_device.unlink(input); + } + this._input_device = value; + }} + private Device _output_device; + public Device output_device { get { return _output_device; } set { + if (output != null) remove_output(output); + if (value != null && receiving) add_output(value.link_sink()); + this._output_device = value; + }} + + public bool created { get; private set; default = false; } + public bool paused { get; private set; default = false; } + private bool push_recv_data = false; + private uint our_ssrc = Random.next_int(); + private int next_seqnum_offset = -1; + private uint32 next_timestamp_offset_base = 0; + private int64 next_timestamp_offset_stamp = 0; + private uint32 next_timestamp_offset { get { + if (next_timestamp_offset_base == 0) return 0; + int64 monotonic_diff = get_monotonic_time() - next_timestamp_offset_stamp; + return next_timestamp_offset_base + (uint32)((double)monotonic_diff / 1000000.0 * payload_type.clockrate); + } } + private uint32 participant_ssrc = 0; + + private Gst.Pad recv_rtcp_sink_pad; + private Gst.Pad recv_rtp_sink_pad; + private Gst.Pad recv_rtp_src_pad; + private Gst.Pad send_rtcp_src_pad; + private Gst.Pad send_rtp_sink_pad; + private Gst.Pad send_rtp_src_pad; + + private Crypto.Srtp.Session? crypto_session = new Crypto.Srtp.Session(); + + public Stream(Plugin plugin, Xmpp.Xep.Jingle.Content content) { + base(content); + this.plugin = plugin; + this.rtpid = plugin.next_free_id(); + + content.notify["senders"].connect_after(on_senders_changed); + } + + public void on_senders_changed() { + if (sending && input == null) { + input_device = input_device; + } + if (receiving && output == null) { + output_device = output_device; + } + } + + public override void create() { + plugin.pause(); + + // Create i/o if needed + + if (input == null && sending) { + input_device = input_device; + } + if (output == null && receiving && media == "audio") { + output_device = output_device; + } + + // Create app elements + send_rtp = Gst.ElementFactory.make("appsink", @"rtp_sink_$rtpid") as Gst.App.Sink; + send_rtp.async = false; + send_rtp.caps = CodecUtil.get_caps(media, payload_type, false); + send_rtp.emit_signals = true; + send_rtp.sync = true; + send_rtp.drop = true; + send_rtp.wait_on_eos = false; + send_rtp.new_sample.connect(on_new_sample); + send_rtp.new_serialized_event.connect(on_new_event); + send_rtp.connect("signal::eos", on_eos_static, this); + pipe.add(send_rtp); + + send_rtcp = Gst.ElementFactory.make("appsink", @"rtcp_sink_$rtpid") as Gst.App.Sink; + send_rtcp.async = false; + send_rtcp.caps = new Gst.Caps.empty_simple("application/x-rtcp"); + send_rtcp.emit_signals = true; + send_rtcp.sync = true; + send_rtcp.drop = true; + send_rtcp.wait_on_eos = false; + send_rtcp.new_sample.connect(on_new_sample); + send_rtcp.connect("signal::eos", on_eos_static, this); + pipe.add(send_rtcp); + + recv_rtp = Gst.ElementFactory.make("appsrc", @"rtp_src_$rtpid") as Gst.App.Src; + recv_rtp.caps = CodecUtil.get_caps(media, payload_type, true); + recv_rtp.do_timestamp = true; + recv_rtp.format = Gst.Format.TIME; + recv_rtp.is_live = true; + pipe.add(recv_rtp); + + recv_rtcp = Gst.ElementFactory.make("appsrc", @"rtcp_src_$rtpid") as Gst.App.Src; + recv_rtcp.caps = new Gst.Caps.empty_simple("application/x-rtcp"); + recv_rtcp.do_timestamp = true; + recv_rtcp.format = Gst.Format.TIME; + recv_rtcp.is_live = true; + pipe.add(recv_rtcp); + + // Connect RTCP + send_rtcp_src_pad = rtpbin.get_request_pad(@"send_rtcp_src_$rtpid"); + send_rtcp_src_pad.link(send_rtcp.get_static_pad("sink")); + recv_rtcp_sink_pad = rtpbin.get_request_pad(@"recv_rtcp_sink_$rtpid"); + recv_rtcp.get_static_pad("src").link(recv_rtcp_sink_pad); + + // Connect input + send_rtp_sink_pad = rtpbin.get_request_pad(@"send_rtp_sink_$rtpid"); + if (input != null) { + input_pad = input.get_request_pad(@"src_$rtpid"); + input_pad.link(send_rtp_sink_pad); + } + + // Connect output + decode = codec_util.get_decode_bin(media, payload_type, @"decode_$rtpid"); + decode_depay = (Gst.RTP.BaseDepayload)((Gst.Bin)decode).get_by_name(@"decode_$(rtpid)_rtp_depay"); + pipe.add(decode); + if (output != null) { + decode.link(output); + } + + // Connect RTP + recv_rtp_sink_pad = rtpbin.get_request_pad(@"recv_rtp_sink_$rtpid"); + recv_rtp.get_static_pad("src").link(recv_rtp_sink_pad); + + created = true; + push_recv_data = true; + plugin.unpause(); + + GLib.Signal.emit_by_name(rtpbin, "get-session", rtpid, out session); + if (session != null && remb_enabled) { + Object internal_session; + session.@get("internal-session", out internal_session); + if (internal_session != null) { + internal_session.connect("signal::on-feedback-rtcp", on_feedback_rtcp, this); + } + Timeout.add(1000, () => remb_adjust()); + } + if (input_device != null && media == "video") { + input_device.update_bitrate(payload_type, target_send_bitrate); + } + } + + private int last_packets_lost = -1; + private uint64 last_packets_received = 0; + private uint64 last_octets_received = 0; + private uint max_target_receive_bitrate = 0; + private int64 last_remb_time = 0; + private bool remb_adjust() { + unowned Gst.Structure? stats; + if (session == null) { + debug("Session for %u finished, turning off remb adjustment", rtpid); + return Source.REMOVE; + } + session.get("stats", out stats); + if (stats == null) { + warning("No stats for session %u", rtpid); + return Source.REMOVE; + } + unowned ValueArray? source_stats; + stats.get("source-stats", typeof(ValueArray), out source_stats); + if (source_stats == null) { + warning("No source-stats for session %u", rtpid); + return Source.REMOVE; + } + + if (input_device == null) return Source.CONTINUE; + + foreach (Value value in source_stats.values) { + unowned Gst.Structure source_stat = (Gst.Structure) value.get_boxed(); + uint32 ssrc; + if (!source_stat.get_uint("ssrc", out ssrc)) continue; + if (ssrc == participant_ssrc) { + int packets_lost; + uint64 packets_received, octets_received; + source_stat.get_int("packets-lost", out packets_lost); + source_stat.get_uint64("packets-received", out packets_received); + source_stat.get_uint64("octets-received", out octets_received); + int new_lost = packets_lost - last_packets_lost; + if (new_lost < 0) new_lost = 0; + uint64 new_received = packets_received - last_packets_received; + if (packets_received < last_packets_received) new_received = 0; + uint64 new_octets = octets_received - last_octets_received; + if (octets_received < last_octets_received) octets_received = 0; + if (new_received == 0) continue; + last_packets_lost = packets_lost; + last_packets_received = packets_received; + last_octets_received = octets_received; + double loss_rate = (double)new_lost / (double)(new_lost + new_received); + uint new_target_receive_bitrate; + if (new_lost <= 0 || loss_rate < 0.02) { + new_target_receive_bitrate = (uint)(1.08 * (double)target_receive_bitrate); + } else if (loss_rate > 0.1) { + new_target_receive_bitrate = (uint)((1.0 - 0.5 * loss_rate) * (double)target_receive_bitrate); + } else { + new_target_receive_bitrate = target_receive_bitrate; + } + if (last_remb_time == 0) { + last_remb_time = get_monotonic_time(); + } else { + int64 time_now = get_monotonic_time(); + int64 time_diff = time_now - last_remb_time; + last_remb_time = time_now; + uint actual_bitrate = (uint)(((double)new_octets * 8.0) * (double)time_diff / 1000.0 / 1000000.0); + new_target_receive_bitrate = uint.max(new_target_receive_bitrate, (uint)(0.9 * (double)actual_bitrate)); + max_target_receive_bitrate = uint.max((uint)(1.5 * (double)actual_bitrate), max_target_receive_bitrate); + new_target_receive_bitrate = uint.min(new_target_receive_bitrate, max_target_receive_bitrate); + } + new_target_receive_bitrate = uint.max(16, new_target_receive_bitrate); // Never go below 16 + if (new_target_receive_bitrate != target_receive_bitrate) { + target_receive_bitrate = new_target_receive_bitrate; + uint8[] data = new uint8[] { + 143, 206, 0, 5, + 0, 0, 0, 0, + 0, 0, 0, 0, + 'R', 'E', 'M', 'B', + 1, 0, 0, 0, + 0, 0, 0, 0 + }; + data[4] = (uint8)((our_ssrc >> 24) & 0xff); + data[5] = (uint8)((our_ssrc >> 16) & 0xff); + data[6] = (uint8)((our_ssrc >> 8) & 0xff); + data[7] = (uint8)(our_ssrc & 0xff); + uint8 br_exp = 0; + uint32 br_mant = target_receive_bitrate * 1000; + uint8 bits = (uint8)Math.log2(br_mant); + if (bits > 16) { + br_exp = (uint8)bits - 16; + br_mant = br_mant >> br_exp; + } + data[17] = (uint8)((br_exp << 2) | ((br_mant >> 16) & 0x3)); + data[18] = (uint8)((br_mant >> 8) & 0xff); + data[19] = (uint8)(br_mant & 0xff); + data[20] = (uint8)((ssrc >> 24) & 0xff); + data[21] = (uint8)((ssrc >> 16) & 0xff); + data[22] = (uint8)((ssrc >> 8) & 0xff); + data[23] = (uint8)(ssrc & 0xff); + encrypt_and_send_rtcp(data); + } + } + } + return Source.CONTINUE; + } + + private static void on_feedback_rtcp(Gst.Element session, uint type, uint fbtype, uint sender_ssrc, uint media_ssrc, Gst.Buffer? fci, Stream self) { + if (self.input_device != null && self.media == "video" && type == 206 && fbtype == 15 && fci != null && sender_ssrc == self.participant_ssrc) { + // https://tools.ietf.org/html/draft-alvestrand-rmcat-remb-03 + uint8[] data; + fci.extract_dup(0, fci.get_size(), out data); + if (data[0] != 'R' || data[1] != 'E' || data[2] != 'M' || data[3] != 'B') return; + uint8 br_exp = data[5] >> 2; + uint32 br_mant = (((uint32)data[5] & 0x3) << 16) + ((uint32)data[6] << 8) + (uint32)data[7]; + self.target_send_bitrate = (br_mant << br_exp) / 1000; + self.input_device.update_bitrate(self.payload_type, self.target_send_bitrate); + } + } + + private void prepare_local_crypto() { + if (local_crypto != null && local_crypto.is_valid && !crypto_session.has_encrypt) { + crypto_session.set_encryption_key(local_crypto.crypto_suite, local_crypto.key, local_crypto.salt); + debug("Setting up encryption with key params %s", local_crypto.key_params); + } + } + + bool flip = false; + uint8 rotation = 0; + + private bool on_new_event(Gst.App.Sink sink) { + if (sink == null || sink != send_rtp) { + return false; + } + Gst.MiniObject obj = sink.try_pull_object(0); + if (obj.type == typeof(Gst.Event)) { + unowned Gst.TagList tags; + if (((Gst.Event)obj).type == Gst.EventType.TAG) { + ((Gst.Event)obj).parse_tag(out tags); + Gst.Video.OrientationMethod orientation_method; + Gst.Video.Orientation.from_tag(tags, out orientation_method); + switch (orientation_method) { + case Gst.Video.OrientationMethod.IDENTITY: + case Gst.Video.OrientationMethod.VERT: + default: + rotation = 0; + break; + case Gst.Video.OrientationMethod.@90R: + case Gst.Video.OrientationMethod.UL_LR: + rotation = 1; + break; + case Gst.Video.OrientationMethod.@180: + case Gst.Video.OrientationMethod.HORIZ: + rotation = 2; + break; + case Gst.Video.OrientationMethod.@90L: + case Gst.Video.OrientationMethod.UR_LL: + rotation = 3; + break; + } + switch (orientation_method) { + case Gst.Video.OrientationMethod.IDENTITY: + case Gst.Video.OrientationMethod.@90R: + case Gst.Video.OrientationMethod.@180: + case Gst.Video.OrientationMethod.@90L: + default: + flip = false; + break; + case Gst.Video.OrientationMethod.VERT: + case Gst.Video.OrientationMethod.UL_LR: + case Gst.Video.OrientationMethod.HORIZ: + case Gst.Video.OrientationMethod.UR_LL: + flip = true; + break; + } + } + } + return false; + } + + + private Gst.FlowReturn on_new_sample(Gst.App.Sink sink) { + if (sink == null) { + debug("Sink is null"); + return Gst.FlowReturn.EOS; + } + if (sink != send_rtp && sink != send_rtcp) { + warning("unknown sample"); + return Gst.FlowReturn.NOT_SUPPORTED; + } + Gst.Sample sample = sink.pull_sample(); + Gst.Buffer buffer = sample.get_buffer(); + if (sink == send_rtp) { + uint buffer_ssrc = 0, buffer_seq = 0; + Gst.RTP.Buffer rtp_buffer; + if (Gst.RTP.Buffer.map(buffer, Gst.MapFlags.READ, out rtp_buffer)) { + buffer_ssrc = rtp_buffer.get_ssrc(); + buffer_seq = rtp_buffer.get_seq(); + next_seqnum_offset = rtp_buffer.get_seq() + 1; + next_timestamp_offset_base = rtp_buffer.get_timestamp(); + next_timestamp_offset_stamp = get_monotonic_time(); + rtp_buffer.unmap(); + } + if (our_ssrc != buffer_ssrc) { + warning_once("Sending RTP %s buffer seq %u with SSRC %u when our ssrc is %u", media, buffer_seq, buffer_ssrc, our_ssrc); + } + } + + if (sink == send_rtp) { + Xmpp.Xep.JingleRtp.HeaderExtension? ext = header_extensions.first_match((it) => it.uri == "urn:3gpp:video-orientation"); + if (ext != null) { + buffer = (Gst.Buffer) buffer.make_writable(); + Gst.RTP.Buffer rtp_buffer; + if (Gst.RTP.Buffer.map(buffer, Gst.MapFlags.WRITE, out rtp_buffer)) { + uint8[] extension_data = new uint8[1]; + bool camera = false; + extension_data[0] = extension_data[0] | (rotation & 0x3); + if (flip) extension_data[0] = extension_data[0] | 0x4; + if (camera) extension_data[0] = extension_data[0] | 0x8; + rtp_buffer.add_extension_onebyte_header(ext.id, extension_data); + } + } + } + + prepare_local_crypto(); + + uint8[] data; + buffer.extract_dup(0, buffer.get_size(), out data); + if (sink == send_rtp) { + encrypt_and_send_rtp((owned) data); + } else if (sink == send_rtcp) { + encrypt_and_send_rtcp((owned) data); + } + return Gst.FlowReturn.OK; + } + + private void encrypt_and_send_rtp(owned uint8[] data) { + Bytes bytes; + if (crypto_session.has_encrypt) { + bytes = new Bytes.take(crypto_session.encrypt_rtp(data)); + } else { + bytes = new Bytes.take(data); + } + on_send_rtp_data(bytes); + } + + private void encrypt_and_send_rtcp(owned uint8[] data) { + Bytes bytes; + if (crypto_session.has_encrypt) { + bytes = new Bytes.take(crypto_session.encrypt_rtcp(data)); + } else { + bytes = new Bytes.take(data); + } + if (rtcp_mux) { + on_send_rtp_data(bytes); + } else { + on_send_rtcp_data(bytes); + } + } + + private static Gst.PadProbeReturn drop_probe() { + return Gst.PadProbeReturn.DROP; + } + + private static void on_eos_static(Gst.App.Sink sink, Stream self) { + debug("EOS on %s", sink.name); + if (sink == self.send_rtp) { + Idle.add(() => { self.on_send_rtp_eos(); return Source.REMOVE; }); + } else if (sink == self.send_rtcp) { + Idle.add(() => { self.on_send_rtcp_eos(); return Source.REMOVE; }); + } + } + + private void on_send_rtp_eos() { + if (send_rtp_src_pad != null) { + send_rtp_src_pad.unlink(send_rtp.get_static_pad("sink")); + send_rtp_src_pad = null; + } + send_rtp.set_locked_state(true); + send_rtp.set_state(Gst.State.NULL); + // This happens async, so pipe might be gone by now. + if (pipe != null) pipe.remove(send_rtp); + send_rtp = null; + debug("Stopped sending RTP for %u", rtpid); + } + + private void on_send_rtcp_eos() { + send_rtcp.set_locked_state(true); + send_rtcp.set_state(Gst.State.NULL); + // This happens async, so pipe might be gone by now. + if (pipe != null) pipe.remove(send_rtcp); + send_rtcp = null; + debug("Stopped sending RTCP for %u", rtpid); + } + + public override void destroy() { + // Stop network communication + push_recv_data = false; + if (recv_rtp != null) recv_rtp.end_of_stream(); + if (recv_rtcp != null) recv_rtcp.end_of_stream(); + if (send_rtp != null) send_rtp.new_sample.disconnect(on_new_sample); + if (send_rtcp != null) send_rtcp.new_sample.disconnect(on_new_sample); + + // Disconnect input device + if (input != null) { + input_pad.unlink(send_rtp_sink_pad); + input.release_request_pad(input_pad); + input_pad = null; + } + if (this._input_device != null) { + if (!paused) this._input_device.unlink(input); + this._input_device = null; + this.input = null; + } + + // Inject EOS + if (send_rtp_sink_pad != null) { + send_rtp_sink_pad.send_event(new Gst.Event.eos()); + } + + // Disconnect decode + if (recv_rtp_src_pad != null) { + recv_rtp_src_pad.add_probe(Gst.PadProbeType.BLOCK, drop_probe); + recv_rtp_src_pad.unlink(decode.get_static_pad("sink")); + } + + // Disconnect output + if (output != null) { + decode.get_static_pad("src").add_probe(Gst.PadProbeType.BLOCK, drop_probe); + decode.unlink(output); + } + + // Disconnect output device + if (this._output_device != null) { + this._output_device.unlink(output); + this._output_device = null; + } + output = null; + + // Destroy decode + if (decode != null) { + decode.set_locked_state(true); + decode.set_state(Gst.State.NULL); + pipe.remove(decode); + decode = null; + decode_depay = null; + } + + // Disconnect and remove RTP input + if (recv_rtp != null) { + recv_rtp.get_static_pad("src").unlink(recv_rtp_sink_pad); + recv_rtp.set_locked_state(true); + recv_rtp.set_state(Gst.State.NULL); + pipe.remove(recv_rtp); + recv_rtp = null; + } + + // Disconnect and remove RTCP input + if (recv_rtcp != null) { + recv_rtcp.get_static_pad("src").unlink(recv_rtcp_sink_pad); + recv_rtcp.set_locked_state(true); + recv_rtcp.set_state(Gst.State.NULL); + pipe.remove(recv_rtcp); + recv_rtcp = null; + } + + // Release rtp pads + if (send_rtp_sink_pad != null) { + rtpbin.release_request_pad(send_rtp_sink_pad); + send_rtp_sink_pad = null; + } + if (recv_rtp_sink_pad != null) { + rtpbin.release_request_pad(recv_rtp_sink_pad); + recv_rtp_sink_pad = null; + } + if (send_rtcp_src_pad != null) { + rtpbin.release_request_pad(send_rtcp_src_pad); + send_rtcp_src_pad = null; + } + if (recv_rtcp_sink_pad != null) { + rtpbin.release_request_pad(recv_rtcp_sink_pad); + recv_rtcp_sink_pad = null; + } + } + + private void prepare_remote_crypto() { + if (remote_crypto != null && remote_crypto.is_valid && !crypto_session.has_decrypt) { + crypto_session.set_decryption_key(remote_crypto.crypto_suite, remote_crypto.key, remote_crypto.salt); + debug("Setting up decryption with key params %s", remote_crypto.key_params); + } + } + + private uint16 previous_incoming_video_orientation_degree = uint16.MAX; + public signal void incoming_video_orientation_changed(uint16 degree); + + public override void on_recv_rtp_data(Bytes bytes) { + if (rtcp_mux && bytes.length >= 2 && bytes.get(1) >= 192 && bytes.get(1) < 224) { + on_recv_rtcp_data(bytes); + return; + } + + if (push_recv_data) { + prepare_remote_crypto(); + + Gst.Buffer buffer; + if (crypto_session.has_decrypt) { + try { + buffer = new Gst.Buffer.wrapped(crypto_session.decrypt_rtp(bytes.get_data())); + } catch (Error e) { + warning("%s (%d)", e.message, e.code); + return; + } + } else { + buffer = new Gst.Buffer.wrapped_bytes(bytes); + } + + Gst.RTP.Buffer rtp_buffer; + if (Gst.RTP.Buffer.map(buffer, Gst.MapFlags.READ, out rtp_buffer)) { + uint buffer_ssrc = rtp_buffer.get_ssrc(); + uint buffer_seq = rtp_buffer.get_seq(); + if (rtp_buffer.get_extension()) { + Xmpp.Xep.JingleRtp.HeaderExtension? ext = header_extensions.first_match((it) => it.uri == "urn:3gpp:video-orientation"); + if (ext != null) { + unowned uint8[] extension_data; + if (rtp_buffer.get_extension_onebyte_header(ext.id, 0, out extension_data) && extension_data.length == 1) { + bool camera = (extension_data[0] & 0x8) > 0; + bool flip = (extension_data[0] & 0x4) > 0; + uint8 rotation = extension_data[0] & 0x3; + uint16 rotation_degree = uint16.MAX; + switch(rotation) { + case 0: rotation_degree = 0; break; + case 1: rotation_degree = 90; break; + case 2: rotation_degree = 180; break; + case 3: rotation_degree = 270; break; + } + if (rotation_degree != previous_incoming_video_orientation_degree) { + incoming_video_orientation_changed(rotation_degree); + previous_incoming_video_orientation_degree = rotation_degree; + } + } + } + } + rtp_buffer.unmap(); + } + + recv_rtp.push_buffer((owned) buffer); + } + } + + public override void on_recv_rtcp_data(Bytes bytes) { + if (push_recv_data) { + prepare_remote_crypto(); + + Gst.Buffer buffer; + if (crypto_session.has_decrypt) { + try { + buffer = new Gst.Buffer.wrapped(crypto_session.decrypt_rtcp(bytes.get_data())); + } catch (Error e) { + warning("%s (%d)", e.message, e.code); + return; + } + } else { + buffer = new Gst.Buffer.wrapped_bytes(bytes); + } + + recv_rtcp.push_buffer((owned) buffer); + } + } + + public override void on_rtp_ready() { + // If full frame has been sent before the connection was ready, the counterpart would only display our video after the next full frame. + // Send a full frame to let the counterpart display our video asap + rtpbin.send_event(new Gst.Event.custom( + Gst.EventType.CUSTOM_UPSTREAM, + new Gst.Structure("GstForceKeyUnit", "all-headers", typeof(bool), true, null)) + ); + } + + public override void on_rtcp_ready() { + int rtp_session_id = (int) rtpid; + uint64 max_delay = int.MAX; + Object rtp_session; + bool rtp_sent; + GLib.Signal.emit_by_name(rtpbin, "get-internal-session", rtp_session_id, out rtp_session); + GLib.Signal.emit_by_name(rtp_session, "send-rtcp-full", max_delay, out rtp_sent); + debug("RTCP is ready, resending rtcp: %s", rtp_sent.to_string()); + } + + public void on_ssrc_pad_added(uint32 ssrc, Gst.Pad pad) { + debug("New ssrc %u with pad %s", ssrc, pad.name); + if (participant_ssrc != 0 && participant_ssrc != ssrc) { + warning("Got second ssrc on stream (old: %u, new: %u), ignoring", participant_ssrc, ssrc); + return; + } + participant_ssrc = ssrc; + recv_rtp_src_pad = pad; + if (decode != null) { + plugin.pause(); + debug("Link %s to %s decode for %s", recv_rtp_src_pad.name, media, name); + recv_rtp_src_pad.link(decode.get_static_pad("sink")); + plugin.unpause(); + } + } + + public void on_send_rtp_src_added(Gst.Pad pad) { + send_rtp_src_pad = pad; + if (send_rtp != null) { + plugin.pause(); + debug("Link %s to %s send_rtp for %s", send_rtp_src_pad.name, media, name); + send_rtp_src_pad.link(send_rtp.get_static_pad("sink")); + plugin.unpause(); + } + } + + public void set_input(Gst.Element? input) { + set_input_and_pause(input, paused); + } + + private void set_input_and_pause(Gst.Element? input, bool paused) { + if (created && this.input != null) { + this.input_pad.unlink(send_rtp_sink_pad); + this.input.release_request_pad(this.input_pad); + this.input_pad = null; + this.input = null; + } + + this.input = input; + this.paused = paused; + + if (created && sending && !paused && input != null) { + plugin.pause(); + input_pad = input.get_request_pad(@"src_$rtpid"); + input_pad.link(send_rtp_sink_pad); + plugin.unpause(); + } + } + + public void pause() { + if (paused) return; + var input = this.input; + set_input_and_pause(null, true); + if (input != null && input_device != null) input_device.unlink(input); + } + + public void unpause() { + if (!paused) return; + set_input_and_pause(input_device != null ? input_device.link_source(payload_type, our_ssrc, next_seqnum_offset, next_timestamp_offset) : null, false); + input_device.update_bitrate(payload_type, target_send_bitrate); + } + + public uint get_participant_ssrc(Xmpp.Jid participant) { + if (participant.equals(content.session.peer_full_jid)) { + return participant_ssrc; + } + return 0; + } + + ulong block_probe_handler_id = 0; + public virtual void add_output(Gst.Element element, Xmpp.Jid? participant = null) { + if (output != null) { + critical("add_output() invoked more than once"); + return; + } + if (participant != null) { + critical("add_output() invoked with participant when not supported"); + return; + } + this.output = element; + if (created) { + plugin.pause(); + decode.link(element); + if (block_probe_handler_id != 0) { + decode.get_static_pad("src").remove_probe(block_probe_handler_id); + } + plugin.unpause(); + } + } + + public virtual void remove_output(Gst.Element element) { + if (output != element) { + critical("remove_output() invoked without prior add_output()"); + return; + } + if (created) { + block_probe_handler_id = decode.get_static_pad("src").add_probe(Gst.PadProbeType.BLOCK, drop_probe); + decode.unlink(element); + } + if (this._output_device != null) { + this._output_device.unlink(element); + this._output_device = null; + } + this.output = null; + } +} + +public class Dino.Plugins.Rtp.VideoStream : Stream { + private Gee.List<Gst.Element> outputs = new ArrayList<Gst.Element>(); + private Gst.Element output_tee; + private Gst.Element rotate; + private ulong incoming_video_orientation_changed_handler; + + public VideoStream(Plugin plugin, Xmpp.Xep.Jingle.Content content) { + base(plugin, content); + if (media != "video") critical("VideoStream created for non-video media"); + } + + public override void create() { + incoming_video_orientation_changed_handler = incoming_video_orientation_changed.connect(on_video_orientation_changed); + plugin.pause(); + rotate = Gst.ElementFactory.make("videoflip", @"video_rotate_$rtpid"); + pipe.add(rotate); + output_tee = Gst.ElementFactory.make("tee", @"video_tee_$rtpid"); + output_tee.@set("allow-not-linked", true); + pipe.add(output_tee); + rotate.link(output_tee); + add_output(rotate); + base.create(); + foreach (Gst.Element output in outputs) { + output_tee.link(output); + } + plugin.unpause(); + } + + private void on_video_orientation_changed(uint16 degree) { + if (rotate != null) { + switch (degree) { + case 0: + rotate.@set("method", 0); + break; + case 90: + rotate.@set("method", 1); + break; + case 180: + rotate.@set("method", 2); + break; + case 270: + rotate.@set("method", 3); + break; + } + } + } + + public override void destroy() { + foreach (Gst.Element output in outputs) { + output_tee.unlink(output); + } + base.destroy(); + rotate.set_locked_state(true); + rotate.set_state(Gst.State.NULL); + rotate.unlink(output_tee); + pipe.remove(rotate); + rotate = null; + output_tee.set_locked_state(true); + output_tee.set_state(Gst.State.NULL); + pipe.remove(output_tee); + output_tee = null; + disconnect(incoming_video_orientation_changed_handler); + } + + public override void add_output(Gst.Element element, Xmpp.Jid? participant) { + if (element == output_tee || element == rotate) { + base.add_output(element); + return; + } + outputs.add(element); + if (output_tee != null) { + output_tee.link(element); + } + } + + public override void remove_output(Gst.Element element) { + if (element == output_tee || element == rotate) { + base.remove_output(element); + return; + } + outputs.remove(element); + if (output_tee != null) { + output_tee.unlink(element); + } + } +} diff --git a/main/src/plugins/rtp/video_widget.vala b/main/src/plugins/rtp/video_widget.vala new file mode 100644 index 00000000..af6eb14e --- /dev/null +++ b/main/src/plugins/rtp/video_widget.vala @@ -0,0 +1,293 @@ +private static extern unowned Gst.Video.Info gst_video_frame_get_video_info(Gst.Video.Frame frame); +[CCode (array_length_type = "size_t", type = "void*")] +private static extern unowned uint8[] gst_video_frame_get_data(Gst.Video.Frame frame); + +public class Dino.Plugins.Rtp.Paintable : Gdk.Paintable, Object { + private Gdk.Paintable image; + private double pixel_aspect_ratio; + + public override Gdk.PaintableFlags get_flags() { + return 0; + } + + public void snapshot(Gdk.Snapshot snapshot, double width, double height) { + if (image != null) image.snapshot(snapshot, width, height); + } + + public override Gdk.Paintable get_current_image() { + if (image != null) return image; + return Gdk.Paintable.empty(0, 0); + } + + public override int get_intrinsic_width() { + if (image != null) return (int) (pixel_aspect_ratio * image.get_intrinsic_width()); + return 0; + } + + public override int get_intrinsic_height() { + if (image != null) return (int) (pixel_aspect_ratio * image.get_intrinsic_height()); + return 0; + } + + public override double get_intrinsic_aspect_ratio() { + if (image != null) return pixel_aspect_ratio * image.get_intrinsic_aspect_ratio(); + return 0.0; + } + + public override void dispose() { + image = null; + base.dispose(); + } + + private void set_paintable(Gdk.Paintable paintable, double pixel_aspect_ratio) { + if (paintable == image) return; + bool size_changed = image == null || + this.pixel_aspect_ratio * image.get_intrinsic_width() != pixel_aspect_ratio * paintable.get_intrinsic_width() || + image.get_intrinsic_height() != paintable.get_intrinsic_height() || + image.get_intrinsic_aspect_ratio() != paintable.get_intrinsic_aspect_ratio(); + + if (image != null) this.image.dispose(); + this.image = paintable; + this.pixel_aspect_ratio = pixel_aspect_ratio; + + if (size_changed) invalidate_size(); + invalidate_contents(); + } + + public void queue_set_texture(Gdk.Texture texture, double pixel_aspect_ratio) { + Idle.add(() => { + set_paintable(texture, pixel_aspect_ratio); + return Source.REMOVE; + }, Priority.DEFAULT); + } +} + +public class Dino.Plugins.Rtp.Sink : Gst.Video.Sink { + internal Paintable paintable = new Paintable(); + private Gst.Video.Info info = new Gst.Video.Info(); + + class construct { + set_metadata("Dino Gtk Video Sink", "Sink/Video", "The video sink used by Dino", "Dino Team <team@dino.im>"); + add_pad_template(new Gst.PadTemplate("sink", Gst.PadDirection.SINK, Gst.PadPresence.ALWAYS, Gst.Caps.from_string(@"video/x-raw, format={ BGRA, ARGB, RGBA, ABGR, RGB, BGR }"))); + } + + construct { + set_drop_out_of_segment(false); + } + + public override bool set_info(Gst.Caps caps, Gst.Video.Info info) { + this.info = info; + return true; + } + + public override void get_times(Gst.Buffer buffer, out Gst.ClockTime start, out Gst.ClockTime end) { + if (buffer.pts != -1) { + start = buffer.pts; + if (buffer.duration != -1) { + end = start + buffer.duration; + } else if (info.fps_n > 0) { + end = start + Gst.Util.uint64_scale_int(Gst.SECOND, info.fps_d, info.fps_n); + } + } + } + + public override Gst.Caps get_caps(Gst.Caps? filter) { + Gst.Caps caps = Gst.Caps.from_string("video/x-raw, format={ BGRA, ARGB, RGBA, ABGR, RGB, BGR }"); + + if (filter != null) { + return filter.intersect(caps, Gst.CapsIntersectMode.FIRST); + } else { + return caps; + } + } + + private Gdk.MemoryFormat memory_format_from_video(Gst.Video.Format format) { + switch (format) { + case Gst.Video.Format.BGRA: return Gdk.MemoryFormat.B8G8R8A8; + case Gst.Video.Format.ARGB: return Gdk.MemoryFormat.A8R8G8B8; + case Gst.Video.Format.RGBA: return Gdk.MemoryFormat.R8G8B8A8; + case Gst.Video.Format.ABGR: return Gdk.MemoryFormat.A8B8G8R8; + case Gst.Video.Format.RGB: return Gdk.MemoryFormat.R8G8B8; + case Gst.Video.Format.BGR: return Gdk.MemoryFormat.B8G8R8; + default: + warning("Unsupported video format: %s", format.to_string()); + return Gdk.MemoryFormat.A8R8G8B8; + } + } + + private Gdk.Texture texture_from_buffer(Gst.Buffer buffer, out double pixel_aspect_ratio) { + Gst.Video.Frame frame = Gst.Video.Frame(); + Gdk.Texture texture; + + if (frame.map(info, buffer, Gst.MapFlags.READ)) { + unowned Gst.Video.Info info = gst_video_frame_get_video_info(frame); + Bytes bytes = new Bytes.take(gst_video_frame_get_data(frame)); + texture = new Gdk.MemoryTexture(info.width, info.height, memory_format_from_video(info.finfo.format), bytes, info.stride[0]); + pixel_aspect_ratio = ((double) info.par_n) / ((double) info.par_d); + frame.unmap(); + } else { + texture = null; + } + return texture; + } + + private void queue_buffer(Gst.Buffer buf) { + double pixel_aspect_ratio; + Gdk.Texture texture = texture_from_buffer(buf, out pixel_aspect_ratio); + if (texture != null) { + paintable.queue_set_texture(texture, pixel_aspect_ratio); + } + } + + public override Gst.FlowReturn show_frame(Gst.Buffer buf) { + @lock.lock(); + queue_buffer(buf); + @lock.unlock(); + + return Gst.FlowReturn.OK; + } +} + +public class Dino.Plugins.Rtp.VideoWidget : Gtk.Widget, Dino.Plugins.VideoCallWidget { + private const int RECAPS_AFTER_CHANGE = 5; + private static uint last_id = 0; + + public uint id { get; private set; } + public Plugin plugin { get; private set; } + public Gst.Pipeline pipe { get { + return plugin.pipe; + }} + + private bool attached; + private Device? connected_device; + private Gst.Element? connected_device_element; + private Stream? connected_stream; + private Gst.Element prepare; + private Gst.Caps last_input_caps; + private Gst.Caps last_caps; + private int recaps_since_change; + private Sink sink; + private Gtk.Picture widget; + + private static uint active_widgets = 0; + + public static bool has_instances() { + return active_widgets > 0; + } + + private static void notify_weak(Object widget_object) { + if (active_widgets > 0) { + debug("Video widget %p destroyed. left=%u", widget_object, active_widgets); + active_widgets--; + } + if (active_widgets == 0 && widget_object is VideoWidget) { + ((VideoWidget)widget_object).plugin.destroy_call_pipe_if_unused(); + } + } + + public VideoWidget(Plugin plugin) { + this.plugin = plugin; + this.layout_manager = new Gtk.BinLayout(); + + id = last_id++; + sink = new Sink() { async = false, sync = true }; + widget = new Gtk.Picture.for_paintable(sink.paintable); + widget.insert_after(this, null); + active_widgets++; + debug("Video widget %p created. total=%u", this, active_widgets); + this.weak_ref(notify_weak); + } + + public void input_caps_changed(GLib.Object pad, ParamSpec spec) { + Gst.Caps? caps = ((Gst.Pad)pad).caps; + if (caps == null) { + debug("Input: No caps"); + return; + } + + int width, height; + caps.get_structure(0).get_int("width", out width); + caps.get_structure(0).get_int("height", out height); + debug("Input resolution changed: %ix%i", width, height); + // Invoke signal on GTK main loop as recipients are likely to use it for doing GTK operations + Idle.add(() => { + resolution_changed(width, height); + return Source.REMOVE; + }); + last_input_caps = caps; + } + + public void display_stream(Xmpp.Xep.JingleRtp.Stream? stream, Xmpp.Jid jid) { + if (sink == null) return; + detach(); + if (stream.media != "video") return; + connected_stream = stream as Stream?; + if (connected_stream == null) return; + plugin.pause(); + pipe.add(sink); + prepare = Gst.parse_bin_from_description(@"videoconvert name=video_widget_$(id)_convert", true); + prepare.name = @"video_widget_$(id)_prepare"; + prepare.get_static_pad("sink").notify["caps"].connect(input_caps_changed); + pipe.add(prepare); + connected_stream.add_output(prepare); + prepare.link(sink); + sink.set_locked_state(false); + plugin.unpause(); + attached = true; + } + + public void display_device(MediaDevice media_device) { + if (sink == null) return; + detach(); + connected_device = media_device as Device; + if (connected_device == null) return; + plugin.pause(); + pipe.add(sink); + prepare = Gst.parse_bin_from_description(@"videoflip video-direction=auto name=video_widget_$(id)_orientation ! videoflip method=horizontal-flip name=video_widget_$(id)_flip ! videoconvert name=video_widget_$(id)_convert", true); + prepare.name = @"video_widget_$(id)_prepare"; + if (prepare is Gst.Bin) { + ((Gst.Bin) prepare).get_by_name(@"video_widget_$(id)_flip").get_static_pad("sink").notify["caps"].connect(input_caps_changed); + } else { + prepare.get_static_pad("sink").notify["caps"].connect(input_caps_changed); + } + pipe.add(prepare); + connected_device_element = connected_device.link_source(); + connected_device_element.link(prepare); + prepare.link(sink); + sink.set_locked_state(false); + plugin.unpause(); + attached = true; + } + + public void detach() { + if (sink == null) return; + if (attached) { + debug("Detaching"); + if (connected_stream != null) { + connected_stream.remove_output(prepare); + connected_stream = null; + } + if (connected_device != null) { + connected_device_element.unlink(sink); + connected_device_element = null; + connected_device.unlink(); + connected_device = null; + } + prepare.set_locked_state(true); + prepare.set_state(Gst.State.NULL); + pipe.remove(prepare); + prepare = null; + sink.set_locked_state(true); + sink.set_state(Gst.State.NULL); + pipe.remove(sink); + attached = false; + } + } + + public override void dispose() { + detach(); + if (widget != null) widget.unparent(); + widget = null; + sink = null; + } +} diff --git a/main/src/plugins/rtp/voice_processor.vala b/main/src/plugins/rtp/voice_processor.vala new file mode 100644 index 00000000..064684c5 --- /dev/null +++ b/main/src/plugins/rtp/voice_processor.vala @@ -0,0 +1,209 @@ +using Gst; + +namespace Dino.Plugins.Rtp { +public static extern Buffer adjust_to_running_time(Base.Transform transform, Buffer buf); +} + +public class Dino.Plugins.Rtp.EchoProbe : Audio.Filter { + private static StaticPadTemplate sink_template = {"sink", PadDirection.SINK, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}}; + private static StaticPadTemplate src_template = {"src", PadDirection.SRC, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}}; + public Audio.Info audio_info { get; private set; } + public signal void on_new_buffer(owned Buffer buffer); + public signal void on_new_delay(int delay); + private uint period_samples; + private uint period_size; + public int delay { get; private set; } + private Base.Adapter adapter = new Base.Adapter(); + + static construct { + add_static_pad_template(sink_template); + add_static_pad_template(src_template); + set_static_metadata("Acoustic Echo Canceller probe", "Generic/Audio", "Gathers playback buffers for echo cancellation", "Dino Team <contact@dino.im>"); + } + + construct { + set_passthrough(true); + } + + public override bool setup(Audio.Info info) { + audio_info = info; + period_samples = info.rate / 100; // 10ms buffers + period_size = period_samples * info.bpf; + return true; + } + + public override bool src_event(owned Event event) { + Query query = new Query.latency(); + if (event.type == EventType.LATENCY && srcpad != null && srcpad.query(query)) { + ClockTime upstream_latency; + query.parse_latency(null, out upstream_latency, null); + int delay = this.delay; + if (upstream_latency != CLOCK_TIME_NONE) { + delay = (int) (upstream_latency / MSECOND); + } else { + delay = 0; + } + if (delay != this.delay) { + debug("Delay adjusted from %ms to %dms", this.delay, delay); + this.delay = delay; + on_new_delay(delay); + } + } + return base.src_event((owned) event); + } + + public override FlowReturn transform_ip(Buffer buf) { + lock (adapter) { + adapter.push(adjust_to_running_time(this, buf)); + while (adapter.available() > period_size) { + on_new_buffer(adapter.take_buffer(period_size)); + } + } + return FlowReturn.OK; + } + + public override bool stop() { + adapter.clear(); + return true; + } +} + +public class Dino.Plugins.Rtp.VoiceProcessor : Audio.Filter { + private static StaticPadTemplate sink_template = {"sink", PadDirection.SINK, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}}; + private static StaticPadTemplate src_template = {"src", PadDirection.SRC, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}}; + public Audio.Info audio_info { get; private set; } + private ulong process_outgoing_buffer_handler_id; + private ulong process_stream_delay_handler_id; + private uint adjust_delay_timeout_id; + private uint period_samples; + private uint period_size; + private Base.Adapter adapter = new Base.Adapter(); + private EchoProbe? echo_probe; + private Audio.StreamVolume? stream_volume; + private ClockTime last_reverse; + private void* native; + + static construct { + add_static_pad_template(sink_template); + add_static_pad_template(src_template); + set_static_metadata("Voice Processor (AGC, AEC, filters, etc.)", "Generic/Audio", "Pre-processes voice with WebRTC Audio Processing Library", "Dino Team <contact@dino.im>"); + } + + construct { + set_passthrough(false); + } + + public VoiceProcessor(EchoProbe? echo_probe = null, Audio.StreamVolume? stream_volume = null) { + this.echo_probe = echo_probe; + this.stream_volume = stream_volume; + } + + private static extern void* init_native(int stream_delay); + private static extern void destroy_native(void* native); + private static extern void analyze_reverse_stream(void* native, Audio.Info info, Buffer buffer); + private static extern void process_stream(void* native, Audio.Info info, Buffer buffer); + private static extern void adjust_stream_delay(void* native); + private static extern void set_stream_delay(void* native, int delay); + private static extern void notify_gain_level(void* native, int gain_level); + private static extern int get_suggested_gain_level(void* native); + private static extern bool get_stream_has_voice(void* native); + + public override bool setup(Audio.Info info) { + debug("VoiceProcessor.setup(%s)", info.to_caps().to_string()); + audio_info = info; + period_samples = info.rate / 100; // 10ms buffers + period_size = period_samples * info.bpf; + adapter.clear(); + return true; + } + + public override bool start() { + native = init_native(echo_probe.delay); + if (process_outgoing_buffer_handler_id == 0 && echo_probe != null) { + process_outgoing_buffer_handler_id = echo_probe.on_new_buffer.connect(process_outgoing_buffer); + } + if (process_stream_delay_handler_id == 0 && echo_probe != null) { + process_stream_delay_handler_id = echo_probe.on_new_delay.connect(process_stream_delay); + } + if (stream_volume == null && sinkpad.get_peer() != null && sinkpad.get_peer().get_parent_element() is Audio.StreamVolume) { + stream_volume = sinkpad.get_peer().get_parent_element() as Audio.StreamVolume; + } + return true; + } + + private bool adjust_delay() { + if (native != null) { + adjust_stream_delay(native); + return Source.CONTINUE; + } else { + adjust_delay_timeout_id = 0; + return Source.REMOVE; + } + } + + private void process_outgoing_buffer(owned Buffer buffer) { + if (buffer.pts != uint64.MAX) { + last_reverse = buffer.pts; + } + if (native != null) { + buffer = (Buffer) buffer.make_writable(); + analyze_reverse_stream(native, echo_probe.audio_info, buffer); + } + if (adjust_delay_timeout_id == 0 && echo_probe != null) { + adjust_delay_timeout_id = Timeout.add(1000, adjust_delay); + } + } + + private void process_stream_delay(int stream_delay) { + if (native != null) { + set_stream_delay(native, stream_delay); + } + } + + public override FlowReturn submit_input_buffer(bool is_discont, Buffer input) { + lock (adapter) { + if (is_discont) { + adapter.clear(); + } + adapter.push(adjust_to_running_time(this, input)); + } + return FlowReturn.OK; + } + + public override FlowReturn generate_output(out Buffer output_buffer) { + lock (adapter) { + if (adapter.available() >= period_size) { + output_buffer = (Gst.Buffer) adapter.take_buffer(period_size).make_writable(); + int old_gain_level = 0; + if (stream_volume != null) { + old_gain_level = (int) (stream_volume.get_volume(Audio.StreamVolumeFormat.LINEAR) * 255.0); + notify_gain_level(native, old_gain_level); + } + process_stream(native, audio_info, output_buffer); + if (stream_volume != null) { + int new_gain_level = get_suggested_gain_level(native); + if (old_gain_level != new_gain_level) { + debug("Gain: %i -> %i", old_gain_level, new_gain_level); + stream_volume.set_volume(Audio.StreamVolumeFormat.LINEAR, ((double)new_gain_level) / 255.0); + } + } + } + } + return FlowReturn.OK; + } + + public override bool stop() { + if (process_outgoing_buffer_handler_id != 0) { + echo_probe.disconnect(process_outgoing_buffer_handler_id); + process_outgoing_buffer_handler_id = 0; + } + if (adjust_delay_timeout_id != 0) { + Source.remove(adjust_delay_timeout_id); + adjust_delay_timeout_id = 0; + } + adapter.clear(); + destroy_native(native); + native = null; + return true; + } +}
\ No newline at end of file diff --git a/main/src/plugins/rtp/voice_processor_native.cpp b/main/src/plugins/rtp/voice_processor_native.cpp new file mode 100644 index 00000000..71cc8a54 --- /dev/null +++ b/main/src/plugins/rtp/voice_processor_native.cpp @@ -0,0 +1,256 @@ +#include <algorithm> +#include <gst/gst.h> +#include <gst/audio/audio.h> +#if defined(WEBRTC0) +#include <webrtc/modules/audio_processing/include/audio_processing.h> +#include <webrtc/modules/interface/module_common_types.h> +#include <webrtc/system_wrappers/include/trace.h> +#elif defined(WEBRTC1) || defined(WEBRTC2) +#include <modules/audio_processing/include/audio_processing.h> +#else +#error "Need to define WEBRTC0, WEBRTC1 or WEBRTC2" +#endif + +#define SAMPLE_RATE 48000 +#define SAMPLE_CHANNELS 1 + +struct _DinoPluginsRtpVoiceProcessorNative { +#if defined(WEBRTC0) + webrtc::AudioProcessing *apm; +#elif defined(WEBRTC1) || defined(WEBRTC2) + rtc::scoped_refptr<webrtc::AudioProcessing> apm; +#endif + gint stream_delay = 0; + gint last_median = 0; + gint last_poor_delays = 0; +}; + +extern "C" void *dino_plugins_rtp_adjust_to_running_time(GstBaseTransform *transform, GstBuffer *buffer) { + GstBuffer *copy = gst_buffer_copy(buffer); + GST_BUFFER_PTS(copy) = gst_segment_to_running_time(&transform->segment, GST_FORMAT_TIME, GST_BUFFER_PTS(buffer)); + return copy; +} + +extern "C" void *dino_plugins_rtp_voice_processor_init_native(gint stream_delay) { + auto *native = new _DinoPluginsRtpVoiceProcessorNative(); + native->stream_delay = stream_delay; +#if defined(WEBRTC0) + webrtc::Config config; + config.Set<webrtc::ExtendedFilter>(new webrtc::ExtendedFilter(true)); + config.Set<webrtc::ExperimentalAgc>(new webrtc::ExperimentalAgc(true, 85)); + native->apm = webrtc::AudioProcessing::Create(config); + + webrtc::AudioProcessing *apm = native->apm; + webrtc::ProcessingConfig pconfig; + pconfig.streams[webrtc::ProcessingConfig::kInputStream] = + webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false); + pconfig.streams[webrtc::ProcessingConfig::kOutputStream] = + webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false); + pconfig.streams[webrtc::ProcessingConfig::kReverseInputStream] = + webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false); + pconfig.streams[webrtc::ProcessingConfig::kReverseOutputStream] = + webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false); + apm->Initialize(pconfig); + apm->high_pass_filter()->Enable(true); + apm->echo_cancellation()->enable_drift_compensation(false); + apm->echo_cancellation()->set_suppression_level(webrtc::EchoCancellation::kModerateSuppression); + apm->echo_cancellation()->enable_delay_logging(true); + apm->echo_cancellation()->Enable(true); + apm->noise_suppression()->set_level(webrtc::NoiseSuppression::kModerate); + apm->noise_suppression()->Enable(true); + apm->gain_control()->set_analog_level_limits(0, 255); + apm->gain_control()->set_mode(webrtc::GainControl::kAdaptiveAnalog); + apm->gain_control()->set_target_level_dbfs(3); + apm->gain_control()->set_compression_gain_db(9); + apm->gain_control()->enable_limiter(true); + apm->gain_control()->Enable(true); + apm->voice_detection()->set_likelihood(webrtc::VoiceDetection::Likelihood::kLowLikelihood); + apm->voice_detection()->Enable(true); +#elif defined(WEBRTC1) || defined(WEBRTC2) + webrtc::AudioProcessing::Config config; + rtc::scoped_refptr<webrtc::AudioProcessing> apm = webrtc::AudioProcessingBuilder().Create(); + native->apm = apm; + config.high_pass_filter.enabled = true; + config.echo_canceller.enabled = true; + config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::Level::kModerate; + config.noise_suppression.enabled = true; + config.gain_controller1.target_level_dbfs = 3; + config.gain_controller1.compression_gain_db = 9; + config.gain_controller1.enable_limiter = true; + config.gain_controller1.enabled = true; +#ifdef WEBRTC1 + config.level_estimation.enabled = true; + config.voice_detection.enabled = true; +#endif + apm->ApplyConfig(config); +#endif + return native; +} + +extern "C" void +dino_plugins_rtp_voice_processor_analyze_reverse_stream(void *native_ptr, GstAudioInfo *info, GstBuffer *buffer) { + auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + int err; + +#if defined(WEBRTC0) + webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false); + webrtc::AudioProcessing *apm = native->apm; + GstMapInfo map; + gst_buffer_map(buffer, &map, GST_MAP_READ); + + webrtc::AudioFrame frame; + frame.num_channels_ = info->channels; + frame.sample_rate_hz_ = info->rate; + frame.samples_per_channel_ = gst_buffer_get_size(buffer) / info->bpf; + memcpy(frame.data_, map.data, frame.samples_per_channel_ * info->bpf); + + apm->set_stream_delay_ms (native->stream_delay); + err = apm->AnalyzeReverseStream(&frame); + + gst_buffer_unmap(buffer, &map); +#elif defined(WEBRTC1) || defined(WEBRTC2) + rtc::scoped_refptr<webrtc::AudioProcessing> apm = native->apm; +#ifdef WEBRTC1 + webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false); +#else + webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS); +#endif + GstAudioBuffer abuf; + if (!gst_audio_buffer_map (&abuf, info, buffer, GST_MAP_READWRITE)) { + g_warning("voice_processor_native.cpp: analyze_reverse_stream: gst_audio_buffer_map failed"); + return; + } + + apm->set_stream_delay_ms (native->stream_delay); + auto * const data = (int16_t * const) abuf.planes[0]; + err = apm->ProcessReverseStream (data, config, config, data); + + gst_audio_buffer_unmap (&abuf); +#endif + if (err < 0) g_warning("voice_processor_native.cpp: ProcessReverseStream %i", err); +} + +extern "C" void dino_plugins_rtp_voice_processor_notify_gain_level(void *native_ptr, gint gain_level) { + auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; +#if defined(WEBRTC0) + webrtc::AudioProcessing *apm = native->apm; + apm->gain_control()->set_stream_analog_level(gain_level); +#elif defined(WEBRTC1) || defined(WEBRTC2) + rtc::scoped_refptr<webrtc::AudioProcessing> apm = native->apm; + apm->set_stream_analog_level(gain_level); +#endif +} + +extern "C" gint dino_plugins_rtp_voice_processor_get_suggested_gain_level(void *native_ptr) { + auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + int level = 0; +#if defined(WEBRTC0) + webrtc::AudioProcessing *apm = native->apm; + level = apm->gain_control()->stream_analog_level(); +#elif defined(WEBRTC1) || defined(WEBRTC2) + rtc::scoped_refptr<webrtc::AudioProcessing> apm = native->apm; + level = apm->recommended_stream_analog_level(); +#endif + return level; +} + +extern "C" bool dino_plugins_rtp_voice_processor_get_stream_has_voice(void *native_ptr) { + auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + bool has_voice = false; +#if defined(WEBRTC0) + webrtc::AudioProcessing *apm = native->apm; + has_voice = apm->voice_detection()->stream_has_voice(); +#elif defined(WEBRTC1) + rtc::scoped_refptr<webrtc::AudioProcessing> apm = native->apm; + webrtc::AudioProcessingStats stats = apm->GetStatistics (); + has_voice = stats.voice_detected.value_or(false); +#endif + return has_voice; +} + +extern "C" void dino_plugins_rtp_voice_processor_set_stream_delay(void *native_ptr, gint stream_delay) { + auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + native->stream_delay = stream_delay; +} + +extern "C" void dino_plugins_rtp_voice_processor_adjust_stream_delay(void *native_ptr) { + auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + int median, std, poor_delays; + float fraction_poor_delays; +#if defined(WEBRTC0) + webrtc::AudioProcessing *apm = native->apm; + apm->echo_cancellation()->GetDelayMetrics(&median, &std, &fraction_poor_delays); + poor_delays = (int)(fraction_poor_delays * 100.0); +#elif defined(WEBRTC1) || defined(WEBRTC2) + rtc::scoped_refptr<webrtc::AudioProcessing> apm = native->apm; + webrtc::AudioProcessingStats stats = apm->GetStatistics(); + median = stats.delay_median_ms.value_or(-1); + std = stats.delay_standard_deviation_ms.value_or(-1); + fraction_poor_delays = (float) stats.divergent_filter_fraction.value_or(-1.0); + poor_delays = (int) (fraction_poor_delays * 100.0); +#endif + if (fraction_poor_delays < 0 || (native->last_median == median && native->last_poor_delays == poor_delays)) return; + g_debug("voice_processor_native.cpp: Stream delay metrics: median=%i std=%i poor_delays=%i%%", median, std, poor_delays); + native->last_median = median; + native->last_poor_delays = poor_delays; +#if defined(WEBRTC0) + if (poor_delays > 90) { + native->stream_delay = std::min(std::max(0, native->stream_delay + std::min(48, std::max(median, -48))), 384); + g_debug("voice_processor_native.cpp: set stream_delay=%i", native->stream_delay); + } +#endif +} + +extern "C" void +dino_plugins_rtp_voice_processor_process_stream(void *native_ptr, GstAudioInfo *info, GstBuffer *buffer) { + auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + int err; + +#if defined(WEBRTC0) + webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false); + webrtc::AudioProcessing *apm = native->apm; + GstMapInfo map; + gst_buffer_map(buffer, &map, GST_MAP_READWRITE); + + webrtc::AudioFrame frame; + frame.num_channels_ = info->channels; + frame.sample_rate_hz_ = info->rate; + frame.samples_per_channel_ = info->rate / 100; + memcpy(frame.data_, map.data, frame.samples_per_channel_ * info->bpf); + + apm->set_stream_delay_ms(native->stream_delay); + err = apm->ProcessStream(&frame); + if (err >= 0) memcpy(map.data, frame.data_, frame.samples_per_channel_ * info->bpf); + + gst_buffer_unmap(buffer, &map); +#elif defined(WEBRTC1) || defined(WEBRTC2) + rtc::scoped_refptr<webrtc::AudioProcessing> apm = native->apm; +#ifdef WEBRTC1 + webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false); +#else + webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS); +#endif + GstAudioBuffer abuf; + if (!gst_audio_buffer_map (&abuf, info, buffer, GST_MAP_READWRITE)) { + g_warning("voice_processor_native.cpp: process_stream: gst_audio_buffer_map failed"); + return; + } + + auto * const data = (int16_t * const) abuf.planes[0]; + err = apm->ProcessStream (data, config, config, data); + + gst_audio_buffer_unmap (&abuf); +#endif + if (err < 0) g_warning("voice_processor_native.cpp: ProcessStream %i", err); +} + +extern "C" void dino_plugins_rtp_voice_processor_destroy_native(void *native_ptr) { + auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; +#if defined(WEBRTC0) + delete native->apm; + native->apm = NULL; +#elif defined(WEBRTC1) || defined(WEBRTC2) + native->apm = nullptr; +#endif + delete native; +}
\ No newline at end of file |
