MT#55283 add TCP keepalives for Redis connections

Change-Id: Id16f7f0f547f45cbef95bc98ede67a30f01cf3b9

daemon/main.c

@@ -89,6 +89,10 @@ struct rtpengine_config rtpe_config = {
 	.kernel_table = -1,
 	.max_sessions = -1,
 	.redis_subscribed_keyspaces = G_QUEUE_INIT,
+	// use aggressive default intervals if enabled for detecting redis service IP failover rapidly
+	// (normally those are internal connections with low jitter and low loss)
+	.redis_tcp_keepalive_intvl = 1,
+	.redis_tcp_keepalive_probes = 3,
 	.redis_expires_secs = 86400,
 	.interfaces = TYPED_GQUEUE_INIT,
 	.homer_protocol = SOCK_DGRAM,
@@ -737,6 +741,10 @@ static void options(int *argc, char ***argv, charp_ht templates) {
 		{ "redis-connect-timeout", 0, 0, G_OPTION_ARG_INT, &rtpe_config.redis_connect_timeout, "Sets a timeout in milliseconds for redis connections", "INT" },
 		{ "redis-format", 0, 0,	G_OPTION_ARG_STRING, &redis_format,		"Format for persistent storage in Redis/KeyDB", "native|bencode|JSON" },
 		{ "subscribe-keyspace", 'k', 0, G_OPTION_ARG_STRING_ARRAY,&ks_a,	"Subscription keyspace list",	"INT INT ..."},
+		{ "redis-tcp-keepalive-time",0,0,G_OPTION_ARG_INT,&rtpe_config.redis_tcp_keepalive_time,"Positive value sets tcp_keepalive_time for redis connections", "INT" },
+		{ "redis-tcp-keepalive-intvl",0,0,G_OPTION_ARG_INT,&rtpe_config.redis_tcp_keepalive_intvl,"Set tcp_keepalive_intvl for redis connections", "INT" },
+		{ "redis-tcp-keepalive-probes",0,0,G_OPTION_ARG_INT,&rtpe_config.redis_tcp_keepalive_probes,"Set tcp_keepalive_probes for redis connections", "INT" },
+
 #if 0
 		// temporarily disabled, see discussion on https://github.com/sipwise/rtpengine/commit/2ebf5a1526c1ce8093b3011a1e23c333b3f99400
 		// related to Change-Id: I83d9b9a844f4f494ad37b44f5d1312f272beff3f

daemon/redis.c

@@ -266,6 +266,21 @@ static int redis_select_db(struct redis *r, int db) {
 	return 0;
 }
 
+void redis_set_keepalive(int fd) {
+	if (rtpe_config.redis_tcp_keepalive_time > 0) {
+		int keepalive_en = 1;
+		setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &keepalive_en, sizeof(keepalive_en));
+
+		setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &rtpe_config.redis_tcp_keepalive_time,
+				sizeof(rtpe_config.redis_tcp_keepalive_time));
+		setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, &rtpe_config.redis_tcp_keepalive_intvl,
+				sizeof(rtpe_config.redis_tcp_keepalive_intvl));
+		setsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT, &rtpe_config.redis_tcp_keepalive_probes,
+				sizeof(rtpe_config.redis_tcp_keepalive_probes));
+	}
+}
+
+
 /* called with r->lock held if necessary */
 static int redis_connect(struct redis *r, int wait, bool resolve) {
 	struct timeval tv;
@@ -302,6 +317,8 @@ static int redis_connect(struct redis *r, int wait, bool resolve) {
 	if (r->ctx->err)
 		goto err2;
 
+	redis_set_keepalive(r->ctx->fd);
+
 	if (redis_set_timeout(r, cmd_timeout))
 		goto err2;
 
@@ -567,6 +584,8 @@ int redis_async_context_alloc(struct redis *r, void *connect_cb, void *disconnec
 		return -1;
 	}
 
+	redis_set_keepalive(r->async_ctx->c.fd);
+
 	// callbacks async context
 	if (redisAsyncSetConnectCallback(r->async_ctx, connect_cb) != REDIS_OK) {
 		rlog(LOG_ERROR, "redis_async_context_alloc: can't set connect callback");
@@ -883,12 +902,14 @@ void redis_notify_loop(void *d) {
 		}
 	}
 
-	// unsubscribe notifications
-	redis_notify_subscribe_action(r, UNSUBSCRIBE_ALL, 0);
+	if (r->state == REDIS_STATE_CONNECTED) {
+		// unsubscribe notifications
+		redis_notify_subscribe_action(r, UNSUBSCRIBE_ALL, 0);
 
-	// free async context
-	redisAsyncDisconnect(r->async_ctx);
-	r->async_ctx = NULL;
+		// free async context
+		redisAsyncDisconnect(r->async_ctx);
+		r->async_ctx = NULL;
+	}
 }
 
 struct redis *redis_new(const endpoint_t *ep, int db, const char *hostname, const char *auth,

docs/rtpengine.md

@@ -651,6 +651,19 @@ call to inject-DTMF won't be sent to __\-\-dtmf-log-dest=__ or __\-\-listen-tcp-
 
     Both formats can be restored from, regardless of this setting.
 
+- __\-\-redis-tcp-keepalive-time=__*INT*
+- __\-\-redis-tcp-keepalive-intvl=__*INT*
+- __\-\-redis-tcp-keepalive-probes=__*INT*
+
+    Controls TCP keepalive behaviour on connections to Redis. The __time__
+    value defaults to zero and must be set to a non-zero positive value to
+    enable keepalives. It's the time in seconds that a socket must have been
+    idle before keepalives are started to be sent.
+
+    __intvl__ is the time in seconds between keepalives, and __probes__ is the
+    number of keepalive probes that are sent before the connection is deemed
+    dead. The defaults are 1 and 3 respectively.
+
 - __-b__, __\-\-b2b-url=__*STRING*
 
     Enables and sets the URI for an XMLRPC callback to be made when a call is

etc/rtpengine.conf

@@ -79,6 +79,9 @@ recording-method = proc
 # redis-cmd-timeout = 0
 # redis-connect-timeout = 1000
 # redis-resolve-on-reconnect = false
+# redis-tcp-keepalive-time = 1
+# redis-tcp-keepalive-intvl = 1
+# redis-tcp-keepalive-probes = 3
 
 # b2b-url = http://127.0.0.1:8090/
 # xmlrpc-format = 0

include/main.h

@@ -57,6 +57,9 @@ enum endpoint_learning {
 	X(redis_connect_timeout) \
 	X(redis_delete_async) \
 	X(redis_delete_async_interval) \
+	X(redis_tcp_keepalive_time) \
+	X(redis_tcp_keepalive_intvl) \
+	X(redis_tcp_keepalive_probes) \
 	X(num_threads) \
 	X(media_num_threads) \
 	X(codec_num_threads) \