TT#172650 update for OpenSSL >= 3.0

closes #1477

Change-Id: I0d5c14f12dd4525d63b435a565f97b5f8abcd81e

daemon/crypto.c

@@ -15,6 +15,7 @@
 #include "rtplib.h"
 #include "rtcplib.h"
 #include "main.h"
+#include "ssllib.h"
 
 
 
@@ -805,6 +806,21 @@ static int aes_f8_encrypt_rtcp(struct crypto_context *c, struct rtcp_packet *r,
 static int hmac_sha1_rtp(struct crypto_context *c, char *out, str *in, uint64_t index) {
 	unsigned char hmac[20];
 	uint32_t roc;
+
+	roc = htonl((index & 0xffffffff0000ULL) >> 16);
+
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+	EVP_MAC_CTX *hc;
+
+	hc = EVP_MAC_CTX_dup(rtpe_hmac_sha1_base);
+	EVP_MAC_init(hc, (unsigned char *) c->session_auth_key,
+			c->params.crypto_suite->srtp_auth_key_len, NULL);
+	EVP_MAC_update(hc, (unsigned char *) in->s, in->len);
+	EVP_MAC_update(hc, (unsigned char *) &roc, sizeof(roc));
+	size_t outsize = sizeof(hmac);
+	EVP_MAC_final(hc, hmac, &outsize, outsize);
+	EVP_MAC_CTX_free(hc);
+#else // <3.0
 	HMAC_CTX *hc;
 
 #if OPENSSL_VERSION_NUMBER >= 0x10100000L
@@ -817,13 +833,13 @@ static int hmac_sha1_rtp(struct crypto_context *c, char *out, str *in, uint64_t
 
 	HMAC_Init_ex(hc, c->session_auth_key, c->params.crypto_suite->srtp_auth_key_len, EVP_sha1(), NULL);
 	HMAC_Update(hc, (unsigned char *) in->s, in->len);
-	roc = htonl((index & 0xffffffff0000ULL) >> 16);
 	HMAC_Update(hc, (unsigned char *) &roc, sizeof(roc));
 	HMAC_Final(hc, hmac, NULL);
 #if OPENSSL_VERSION_NUMBER >= 0x10100000L
 	HMAC_CTX_free(hc);
 #else
 	HMAC_CTX_cleanup(hc);
+#endif
 #endif
 
 	assert(sizeof(hmac) >= c->params.crypto_suite->srtp_auth_tag);

daemon/dtls.c

@@ -187,9 +187,12 @@ static void dump_cert(struct dtls_cert *cert) {
 static int cert_init(void) {
 	X509 *x509 = NULL;
 	EVP_PKEY *pkey = NULL;
-	BIGNUM *exponent = NULL, *serial_number = NULL;
+	BIGNUM *serial_number = NULL;
+#if OPENSSL_VERSION_NUMBER < 0x30000000L
 	RSA *rsa = NULL;
 	EC_KEY *ec_key = NULL;
+	BIGNUM *exponent = NULL;
+#endif
 	ASN1_INTEGER *asn1_serial_number;
 	X509_NAME *name;
 	struct dtls_cert *new_cert;
@@ -198,12 +201,17 @@ static int cert_init(void) {
 
 	/* objects */
 
-	pkey = EVP_PKEY_new();
 	serial_number = BN_new();
 	name = X509_NAME_new();
 	x509 = X509_new();
-	if (!pkey || !serial_number || !name || !x509)
+	if (!serial_number || !name || !x509)
+		goto err;
+
+#if OPENSSL_VERSION_NUMBER < 0x30000000L
+	pkey = EVP_PKEY_new();
+	if (!pkey)
 		goto err;
+#endif
 
 	/* key */
 
@@ -211,9 +219,11 @@ static int cert_init(void) {
 		ilogs(crypto, LOG_DEBUG, "Using %i-bit RSA key for DTLS certificate",
 				rtpe_config.dtls_rsa_key_size);
 
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+		pkey = EVP_RSA_gen(rtpe_config.dtls_rsa_key_size);
+#else // <3.0
 		exponent = BN_new();
 		rsa = RSA_new();
-
 		if (!exponent || !rsa)
 			goto err;
 
@@ -226,10 +236,15 @@ static int cert_init(void) {
 		if (!EVP_PKEY_assign_RSA(pkey, rsa))
 			goto err;
 		rsa = NULL;
+#endif
+
 	}
 	else if (rtpe_config.dtls_cert_cipher == DCC_EC_PRIME256v1) {
 		ilogs(crypto, LOG_DEBUG, "Using EC-prime256v1 key for DTLS certificate");
 
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+		pkey = EVP_EC_gen("prime256v1");
+#else
 		ec_key = EC_KEY_new_by_curve_name(NID_X9_62_prime256v1);
 
 		if (!ec_key)
@@ -241,10 +256,15 @@ static int cert_init(void) {
 		if (!EVP_PKEY_assign_EC_KEY(pkey, ec_key))
 			goto err;
 		ec_key = NULL;
+#endif
 	}
 	else
 		abort();
 
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+	if (!pkey)
+		goto err;
+#endif
 	/* x509 cert */
 
 	if (!X509_set_pubkey(x509, pkey))
@@ -252,8 +272,13 @@ static int cert_init(void) {
 
 	/* serial */
 
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
 	if (!BN_pseudo_rand(serial_number, 64, 0, 0))
 		goto err;
+#else
+	if (!BN_rand(serial_number, 64, 0, 0))
+		goto err;
+#endif
 
 	asn1_serial_number = X509_get_serialNumber(x509);
 	if (!asn1_serial_number)
@@ -323,7 +348,9 @@ static int cert_init(void) {
 
 	/* cleanup */
 
+#if OPENSSL_VERSION_NUMBER < 0x30000000L
 	BN_free(exponent);
+#endif
 	BN_free(serial_number);
 	X509_NAME_free(name);
 
@@ -334,12 +361,14 @@ err:
 
 	if (pkey)
 		EVP_PKEY_free(pkey);
+#if OPENSSL_VERSION_NUMBER < 0x30000000L
 	if (exponent)
 		BN_free(exponent);
 	if (rsa)
 		RSA_free(rsa);
 	if (ec_key)
 		EC_KEY_free(ec_key);
+#endif
 	if (x509)
 		X509_free(x509);
 	if (serial_number)
@@ -606,12 +635,17 @@ int dtls_connection_init(struct dtls_connection *d, struct packet_stream *ps, in
 	d->init = 1;
 	SSL_set_mode(d->ssl, SSL_MODE_ENABLE_PARTIAL_WRITE | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
 
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+	int ec_groups[1] = { NID_X9_62_prime256v1 };
+	SSL_set1_groups(d->ssl, &ec_groups, G_N_ELEMENTS(ec_groups));
+#else // <3.0
 	EC_KEY* ecdh = EC_KEY_new_by_curve_name(NID_X9_62_prime256v1);
 	if (ecdh == NULL)
 		goto error;
 	SSL_set_options(d->ssl, SSL_OP_SINGLE_ECDH_USE);
 	SSL_set_tmp_ecdh(d->ssl, ecdh);
 	EC_KEY_free(ecdh);
+#endif
 
 #if defined(SSL_OP_NO_QUERY_MTU)
 	SSL_CTX_set_options(d->ssl_ctx, SSL_OP_NO_QUERY_MTU);

daemon/rtpengine.pod

@@ -925,6 +925,13 @@ guaranteed that only a single thread will ever read from a particular socket,
 thus maintaining the order of the packets. Might help when having issues with
 DTMF packets (RFC 2833).
 
+=item B<--dtls-cert-cipher=>B<prime256v1>|B<RSA>
+
+Choose the type of key to use for the signature used by the self-signed
+certificate used for DTLS. The previous default was B<RSA>. The current default
+and the only other option is B<prime256v1> which is a 256-bit elliptic-curve
+key.
+
 =item B<--dtls-signature=>B<SHA-256>|B<SHA-1>
 
 Choose the hash algorithm to use for the signature used by the self-signed

daemon/stun.c

@@ -13,6 +13,7 @@
 #include "aux.h"
 #include "log.h"
 #include "ice.h"
+#include "ssllib.h"
 
 
 
@@ -339,6 +340,20 @@ static void fingerprint(struct msghdr *mh, struct fingerprint *fp) {
 
 static void __integrity(struct iovec *iov, int iov_cnt, str *pwd, char *digest) {
 	int i;
+
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+	EVP_MAC_CTX *ctx;
+
+	ctx = EVP_MAC_CTX_dup(rtpe_hmac_sha1_base);
+	EVP_MAC_init(ctx, (unsigned char *) pwd->s, pwd->len, NULL);
+
+	for (i = 0; i < iov_cnt; i++)
+		EVP_MAC_update(ctx, iov[i].iov_base, iov[i].iov_len);
+
+	size_t outsize = 20;
+	EVP_MAC_final(ctx, (unsigned char *) digest, &outsize, outsize);
+	EVP_MAC_CTX_free(ctx);
+#else // <3.0
 	HMAC_CTX *ctx;
 
 #if OPENSSL_VERSION_NUMBER >= 0x10100000L
@@ -360,6 +375,7 @@ static void __integrity(struct iovec *iov, int iov_cnt, str *pwd, char *digest)
 #else
 	HMAC_CTX_cleanup(ctx);
 #endif
+#endif
 }
 
 static void integrity(struct msghdr *mh, struct msg_integrity *mi, str *pwd) {

include/main.h

@@ -106,8 +106,8 @@ struct rtpengine_config {
 	int                     jb_length;
 	int                     jb_clock_drift;
 	enum {
-		DCC_RSA = 0,
-		DCC_EC_PRIME256v1,
+		DCC_EC_PRIME256v1 = 0,
+		DCC_RSA,
 	}			dtls_cert_cipher;
 	int			dtls_rsa_key_size;
 	int			dtls_mtu;

lib/ssllib.c

@@ -4,6 +4,12 @@
 #include "auxlib.h"
 
 
+
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+EVP_MAC_CTX *rtpe_hmac_sha1_base;
+#endif
+
+
 #if OPENSSL_VERSION_NUMBER < 0x10100000L
 static mutex_t *openssl_locks;
 
@@ -44,4 +50,17 @@ void rtpe_ssl_init(void) {
 	SSL_load_error_strings();
 	make_OpenSSL_thread_safe();
 #endif
+
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+	EVP_MAC *rtpe_evp_hmac = EVP_MAC_fetch(NULL, "hmac", NULL);
+	assert(rtpe_evp_hmac != NULL);
+
+	rtpe_hmac_sha1_base = EVP_MAC_CTX_new(rtpe_evp_hmac);
+	assert(rtpe_hmac_sha1_base != NULL);
+	static const OSSL_PARAM params[2] = {
+		OSSL_PARAM_utf8_string("digest", "sha-1", 5),
+		OSSL_PARAM_END,
+	};
+	EVP_MAC_CTX_set_params(rtpe_hmac_sha1_base, params);
+#endif
 }

lib/ssllib.h

@@ -2,6 +2,16 @@
 #define __SSLLIB_H__
 
 
+#include <openssl/ssl.h>
+
+
+
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+extern EVP_MAC_CTX *rtpe_hmac_sha1_base;
+#endif
+
+
+
 void rtpe_ssl_init(void);
 
 

t/aes-crypt.c

@@ -5,6 +5,7 @@
 #include "rtplib.h"
 #include "log.h"
 #include "main.h"
+#include "ssllib.h"
 
 struct rtpengine_config rtpe_config;
 
@@ -206,6 +207,7 @@ int main(int argc, char** argv) {
 	struct crypto_context ctx, ctx2;
 
 	crypto_init_main();
+	rtpe_ssl_init();
 	
 	str_init(&suite, "AES_CM_128_HMAC_SHA1_80");
 	c = crypto_find_suite(&suite);