Implements redis onekey concept

The redis onekey concepts is introduced to reduce traffic to redis
and redis notification traffic.
It modifies the current structure for one call in redis, which are
multiple keys with pre- and postfixes and the callid in between to
one key with the structure "json-<callid>". The value is a json
formatted string with the previous multi-key identifiers in it.

README.md

@@ -175,7 +175,8 @@ option and which are reproduced below:
 	  -w, --redis-write=[PW@]IP:PORT/INT Connect to Redis write database
 	  -k, --subscribe-keyspace         Subscription keyspace list
 	  --redis-num-threads=INT          Number of Redis restore threads
-	  --redis-expires=INT              Expire time in seconds for redis keys	 
+	  --redis-expires=INT              Expire time in seconds for redis keys
+	  --redis-multikey=INT             Use multiple redis keys for storing the call (old behaviour) DEPRECATED
 	  -q, --no-redis-required          Start even if can't connect to redis databases
 	  -b, --b2b-url=STRING             XMLRPC URL of B2B UA
 	  -L, --log-level=INT              Mask log priorities above this level
@@ -426,6 +427,10 @@ The options are described in more detail below.
 
         Expire time in seconds for redis keys. Default is 86400.
 
+*  --redis-multikey
+
+	Use multiple redis keys for storing the call (old behaviour) DEPRECATED
+
 *  -q, --no-redis-required
 	When this paramter is present or NO_REDIS_REQUIRED='yes' or '1' in config file, rtpengine starts even
 	if there is no initial connection to redis databases(either to -r or to -w or to both redis).

daemon/Makefile

@@ -8,6 +8,7 @@ CFLAGS+=	`pkg-config --cflags zlib`
 CFLAGS+=	`pkg-config --cflags openssl`
 CFLAGS+=	`pkg-config --cflags libevent_pthreads`
 CFLAGS+=	`pcre-config --cflags`
+CFLAGS+=	`pkg-config --cflags json-glib-1.0`
 CFLAGS+=	-I. -I../kernel-module/ -I../lib/
 CFLAGS+=	-D_GNU_SOURCE
 
@@ -26,10 +27,13 @@ LDFLAGS+=       `pkg-config --libs libpcre`
 LDFLAGS+=       `pkg-config --libs libcrypto`
 LDFLAGS+=	`pkg-config --libs openssl`
 LDFLAGS+=	`pkg-config --libs libevent_pthreads`
+LDFLAGS+=	`pkg-config --libs`
 LDFLAGS+=	-lpcap
 LDFLAGS+=	`pcre-config --libs`
 LDFLAGS+=	`xmlrpc-c-config client --libs`
 LDFLAGS+=	-lhiredis
+LDFLAGS+=       `pkg-config --libs libcurl`
+LDFLAGS+=	`pkg-config --libs json-glib-1.0`
 
 include ../lib/lib.Makefile
 

daemon/call.c

@@ -594,8 +594,13 @@ static void callmaster_timer(void *ptr) {
 
 		rwlock_unlock_r(&sfd->call->master_lock);
 
-		if (update)
-			redis_update(ps->call, m->conf.redis_write);
+		if (update) {
+			if (m->conf.redis_multikey) {
+				redis_update(ps->call, m->conf.redis_write);
+			} else {
+				redis_update_onekey(ps->call, m->conf.redis_write);
+			}
+		}
 
 next:
 		g_hash_table_remove(hlp.addr_sfd, &ep);

daemon/call.h

@@ -5,7 +5,8 @@
 
 /* XXX split everything into call_signalling.[ch] and call_packets.[ch] or w/e */
 
-
+#include <glib-object.h>
+#include <json-glib/json-glib.h>
 
 #include <sys/types.h>
 #include <glib.h>
@@ -443,6 +444,7 @@ struct call {
 	unsigned int		foreign_call; // created_via_redis_notify call
 
 	struct recording 	*recording;
+	JsonReader *root_reader;
 };
 
 struct callmaster_config {
@@ -460,7 +462,8 @@ struct callmaster_config {
 	struct event_base   *redis_notify_event_base;
 	GQueue		        *redis_subscribed_keyspaces;
 	struct redisAsyncContext *redis_notify_async_context;
-	unsigned int            redis_expires_secs;
+	unsigned int        redis_expires_secs;
+	unsigned int		redis_multikey;
 	char			*b2b_url;
 	unsigned char		default_tos;
 	enum xmlrpc_format	fmt;

daemon/call_interfaces.c

@@ -186,7 +186,11 @@ static str *call_update_lookup_udp(char **out, struct callmaster *m, enum call_o
 			sp.index, sp.index, out[RE_UDP_COOKIE], SAF_UDP);
 	rwlock_unlock_w(&c->master_lock);
 
-	redis_update(c, m->conf.redis_write);
+	if (m->conf.redis_multikey) {
+		redis_update(c, m->conf.redis_write);
+	} else {
+		redis_update_onekey(c, m->conf.redis_write);
+	}
 
 	gettimeofday(&(monologue->started), NULL);
 
@@ -334,7 +338,11 @@ out2:
 	rwlock_unlock_w(&c->master_lock);
 	streams_free(&s);
 
-	redis_update(c, m->conf.redis_write);
+	if (m->conf.redis_multikey) {
+		redis_update(c, m->conf.redis_write);
+	} else {
+		redis_update_onekey(c, m->conf.redis_write);
+	}
 
 	ilog(LOG_INFO, "Returning to SIP proxy: "STR_FORMAT"", STR_FMT0(ret));
 	obj_put(c);
@@ -764,10 +772,16 @@ static const char *call_offer_answer_ng(bencode_item_t *input, struct callmaster
 	}
 
 	rwlock_unlock_w(&call->master_lock);
-	if (!flags.no_redis_update)
-		redis_update(call, m->conf.redis_write);
-	else
+
+	if (!flags.no_redis_update) {
+		if (m->conf.redis_multikey) {
+			redis_update(call, m->conf.redis_write);
+		} else {
+			redis_update_onekey(call,m->conf.redis_write);
+		}
+	} else {
 		ilog(LOG_DEBUG, "Not updating Redis due to present no-redis-update flag");
+	}
 	obj_put(call);
 
 	gettimeofday(&(monologue->started), NULL);

daemon/main.c

@@ -62,6 +62,7 @@ static unsigned int timeout;
 static unsigned int silent_timeout;
 static unsigned int final_timeout;
 static unsigned int redis_expires = 86400;
+static unsigned int redis_multikey = 0;
 static int port_min = 30000;
 static int port_max = 40000;
 static int max_sessions = -1;
@@ -298,6 +299,7 @@ static void options(int *argc, char ***argv) {
 		{ "redis-write",'w', 0, G_OPTION_ARG_STRING,    &redisps_write, "Connect to Redis write database",      "[PW@]IP:PORT/INT"       },
 		{ "redis-num-threads", 0, 0, G_OPTION_ARG_INT, &redis_num_threads, "Number of Redis restore threads",      "INT"       },
 		{ "redis-expires", 0, 0, G_OPTION_ARG_INT, &redis_expires, "Expire time in seconds for redis keys",      "INT"       },
+		{ "redis-multikey", 0, 0, G_OPTION_ARG_INT, &redis_multikey, "Use multiple redis keys for storing the call (old behaviour) DEPRECATED", "INT" },
 		{ "no-redis-required", 'q', 0, G_OPTION_ARG_NONE, &no_redis_required, "Start no matter of redis connection state", NULL },
 		{ "b2b-url",	'b', 0, G_OPTION_ARG_STRING,	&b2b_url,	"XMLRPC URL of B2B UA"	,	"STRING"	},
 		{ "log-facility",0,  0,	G_OPTION_ARG_STRING, &log_facility_s, "Syslog facility to use for logging", "daemon|local0|...|local7"},
@@ -621,6 +623,7 @@ no_kernel:
 	}
 
 	mc.redis_expires_secs = redis_expires;
+	mc.redis_multikey = redis_multikey;
 
 	ctx->m->conf = mc;
 

daemon/media_socket.c

@@ -1471,8 +1471,13 @@ static void stream_fd_readable(int fd, void *p, uintptr_t u) {
 out:
 	ca = sfd->call ? : NULL;
 
-	if (ca && update)
-		redis_update(ca, ca->callmaster->conf.redis_write);
+	if (ca && update) {
+		if (ca->callmaster->conf.redis_multikey) {
+			redis_update(ca, ca->callmaster->conf.redis_write);
+		} else {
+			redis_update_onekey(ca, ca->callmaster->conf.redis_write);
+		}
+	}
 done:
 	log_info_clear();
 }

daemon/redis.h

@@ -91,23 +91,17 @@ INLINE gboolean g_hash_table_insert_check(GHashTable *h, gpointer k, gpointer v)
 #endif
 
 
-
-
-
-
 #define rlog(l, x...) ilog(l | LOG_FLAG_RESTORE, x)
 
-
-
 #define REDIS_FMT(x) (x)->len, (x)->str
 
-
 void redis_notify_loop(void *d);
 
 
 struct redis *redis_new(const endpoint_t *, int, const char *, enum redis_role, int no_redis_required);
 int redis_restore(struct callmaster *, struct redis *);
 void redis_update(struct call *, struct redis *);
+void redis_update_onekey(struct call *c, struct redis *r);
 void redis_delete(struct call *, struct redis *);
 void redis_wipe(struct redis *);
 int redis_notify_event_base_action(struct callmaster *cm, enum event_base_action);

debian/control

@@ -13,6 +13,7 @@ Build-Depends: debhelper (>= 5),
                libevent-dev (>= 2.0),
                libglib2.0-dev (>= 2.30),
                libhiredis-dev,
+               libjson-glib-dev,
                libpcap0.8-dev | libpcap-dev,
                libpcre3-dev,
                libssl-dev (>= 1.0.1),

debian/ngcp-rtpengine-daemon.init

@@ -74,6 +74,7 @@ fi
 [ -z "$REDIS_WRITE_AUTH_PW" ] || export RTPENGINE_REDIS_WRITE_AUTH_PW="$REDIS_WRITE_AUTH_PW"
 [ -z "$REDIS_NUM_THREADS" ] || OPTIONS="$OPTIONS --redis-num-threads=$REDIS_NUM_THREADS"
 [ -z "$REDIS_EXPIRES" ] || OPTIONS="$OPTIONS --redis-expires=$REDIS_EXPIRES"
+[ -z "$REDIS_MULTIKEY" ] || OPTIONS="$OPTIONS --redis-multikey=$REDIS_MULTIKEY"
 [ -z "$NO_REDIS_REQUIRED" -o \( "$NO_REDIS_REQUIRED" != "1" -a "$NO_REDIS_REQUIRED" != "yes" \) ] || OPTIONS="$OPTIONS --no-redis-required"
 [ -z "$B2B_URL" ] || OPTIONS="$OPTIONS --b2b-url=$B2B_URL"
 [ -z "$NO_FALLBACK" -o \( "$NO_FALLBACK" != "1" -a "$NO_FALLBACK" != "yes" \) ] || OPTIONS="$OPTIONS --no-fallback"