TT#144701 support JSON in NG protocol

Change-Id: I5ffe551c2712d0dece3490cd5653c6817603642d

README.md

@@ -568,22 +568,31 @@ Call recording can be accomplished in one of two ways:
 The *ng* Control Protocol
 =========================
 
-In order to enable several advanced features in *rtpengine*, a new advanced control protocol has been devised
-which passes the complete SDP body from the SIP proxy to the *rtpengine* daemon, has the body rewritten in
-the daemon, and then passed back to the SIP proxy to embed into the SIP message.
-
-This control protocol is based on the [bencode](http://en.wikipedia.org/wiki/Bencode) standard and runs over
-UDP transport. *Bencoding* supports a similar feature set as the more popular JSON encoding (dictionaries/hashes,
-lists/arrays, arbitrary byte strings) but offers some benefits over JSON encoding, e.g. simpler and more efficient
-encoding, less encoding overhead, deterministic encoding and faster encoding and decoding. A disadvantage over
-JSON is that it's not a readily human readable format.
-
-Each message passed between the SIP proxy and the media proxy contains of two parts: a message cookie, and a
-bencoded dictionary, separated by a single space. The message cookie serves the same purpose as in the control
-protocol used by *Kamailio*'s *rtpproxy* module: matching requests to responses, and retransmission detection.
-The message cookie in the response generated to a particular request therefore must be the same as in the
+In order to enable several advanced features in *rtpengine*, a new advanced
+control protocol has been devised which passes the complete SDP body from the
+SIP proxy to the *rtpengine* daemon, has the body rewritten in the daemon, and
+then passed back to the SIP proxy to embed into the SIP message.
+
+This control protocol is supported over a number of different transports (plain
+UDP, plain TCP, HTTP, WebSocket) and loosely follows the same format as used by
+*Kamailio*'s *rtpproxy* module. Each message passed between the SIP proxy and
+the media proxy contains of two parts: a unique message cookie and a dictionary
+document, separated by a single space. The message cookie is used to match
+requests to responses and to detect retransmissions. The message cookie in the
+response generated to a particular request therefore must be the same as in the
 request.
 
+The dictionary document can be in one of two formats. It can be a JSON object
+or it can be a dictionary in [bencode](http://en.wikipedia.org/wiki/Bencode)
+format. *Bencoding* supports a subset of the features of JSON
+(dictionaries/hashes, lists/arrays, arbitrary byte strings) but offers some
+benefits over JSON encoding, e.g. simpler and more efficient encoding, less
+encoding overhead, deterministic encoding and faster encoding and decoding.
+Disadvantages compared to JSON are that it's not a readily human readable
+format and that support in programming languages might be difficult to come by.
+Internally *rtpengine* uses *bencoding* natively, leading to additional
+overhead when JSON is in use as it has to be converted.
+
 The dictionary of each request must contain at least one key called `command`. The corresponding value must be
 a string and determines the type of message. Currently the following commands are defined:
 
@@ -623,7 +632,8 @@ For example, a `ping` message and its corresponding `pong` reply would be writte
 	{ "command": "ping" }
 	{ "result": "pong" }
 
-While the actual messages as encoded on the wire, including the message cookie, might look like this:
+While the actual messages as encoded on the wire, including the message cookie,
+might look like this in *bencode* format:
 
 	5323_1 d7:command4:pinge
 	5323_1 d6:result4:ponge
@@ -631,7 +641,13 @@ While the actual messages as encoded on the wire, including the message cookie,
 All keys and values are case-sensitive unless specified otherwise. The requirement stipulated by the *bencode*
 standard that dictionary keys must be present in lexicographical order is not currently honoured.
 
-The *ng* protocol is used by *Kamailio*'s *rtpengine* module, which is based on the older module called *rtpproxy-ng*.
+The *ng* protocol is used by *Kamailio*'s *rtpengine* module, which is based on
+the older module called *rtpproxy-ng*, and utilises *bencoding* and the UDP
+transport by default, or alternatively WebSocket if so configured.
+
+Of course the agent controlling *rtpengine* via the *ng* protocol does not have
+to be a SIP proxy. Any process that involves SDP can potentially talk to
+*rtpengine* via this protocol.
 
 `ping` Message
 --------------
@@ -817,7 +833,6 @@ Optionally included keys are:
 
 		Legacy alias to SDES=pad.
 
-
 	- `generate mid`
 
 		Add `a=mid` attributes to the outgoing SDP if they were not already present.

daemon/bencode.c

@@ -6,6 +6,7 @@
 #include <assert.h>
 #include <string.h>
 #include <ctype.h>
+#include <json-glib/json-glib.h>
 
 /* set to 0 for alloc debugging, e.g. through valgrind */
 #define BENCODE_MIN_BUFFER_PIECE_LEN	512
@@ -261,6 +262,7 @@ bencode_item_t *bencode_integer(bencode_buffer_t *buf, long long int i) {
 	ret->iov[1].iov_len = 0;
 	ret->iov_cnt = 1;
 	ret->str_len = rlen;
+	ret->value = i;
 
 	return ret;
 }
@@ -804,3 +806,180 @@ static ssize_t __bencode_next(const char *s, ssize_t offset, size_t len) {
 ssize_t bencode_valid(const char *s, size_t len) {
 	return __bencode_next(s, 0, len);
 }
+
+
+
+static bencode_item_t *bencode_convert_json_node(bencode_buffer_t *buf, JsonNode *node);
+
+static bencode_item_t *bencode_convert_json_dict(bencode_buffer_t *buf, JsonNode *node) {
+	JsonObject *obj = json_node_get_object(node);
+	if (!obj)
+		return NULL;
+	bencode_item_t *dict = bencode_dictionary(buf);
+	if (!dict)
+		return NULL;
+
+	JsonObjectIter iter;
+	json_object_iter_init(&iter, obj);
+	const char *key;
+	JsonNode *value;
+	while (json_object_iter_next(&iter, &key, &value)) {
+		if (!key || !value)
+			return NULL;
+		bencode_item_t *b_val = bencode_convert_json_node(buf, value);
+		if (!b_val)
+			return NULL;
+		bencode_dictionary_add(dict, key, b_val);
+	}
+	return dict;
+}
+
+static bencode_item_t *bencode_convert_json_array(bencode_buffer_t *buf, JsonNode *node) {
+	JsonArray *arr = json_node_get_array(node);
+	if (!arr)
+		return NULL;
+	bencode_item_t *list = bencode_list(buf);
+	if (!list)
+		return NULL;
+	guint len = json_array_get_length(arr);
+	for (guint i = 0; i < len; i++) {
+		JsonNode *el = json_array_get_element(arr, i);
+		if (!el)
+			return NULL;
+		bencode_item_t *it = bencode_convert_json_node(buf, el);
+		if (!it)
+			return NULL;
+		bencode_list_add(list, it);
+	}
+	return list;
+}
+
+static bencode_item_t *bencode_convert_json_value(bencode_buffer_t *buf, JsonNode *node) {
+	GType type = json_node_get_value_type(node);
+	switch (type) {
+		case G_TYPE_STRING:;
+			const char *s = json_node_get_string(node);
+			if (!s)
+				return NULL;
+			return bencode_string(buf, s);
+		case G_TYPE_INT:
+		case G_TYPE_UINT:
+		case G_TYPE_LONG:
+		case G_TYPE_ULONG:
+		case G_TYPE_INT64:
+		case G_TYPE_UINT64:
+		case G_TYPE_BOOLEAN:;
+			gint64 i = json_node_get_int(node);
+			return bencode_integer(buf, i);
+		// everything else is unsupported
+	}
+	return NULL;
+}
+
+static bencode_item_t *bencode_convert_json_node(bencode_buffer_t *buf, JsonNode *node) {
+	JsonNodeType type = json_node_get_node_type(node);
+	switch (type) {
+		case JSON_NODE_OBJECT:
+			return bencode_convert_json_dict(buf, node);
+		case JSON_NODE_ARRAY:
+			return bencode_convert_json_array(buf, node);
+		case JSON_NODE_VALUE:
+			return bencode_convert_json_value(buf, node);
+		default:
+			return NULL;
+	}
+}
+
+bencode_item_t *bencode_convert_json(bencode_buffer_t *buf, JsonParser *json) {
+	JsonNode *root = json_parser_get_root(json);
+	if (!root)
+		return NULL;
+	return bencode_convert_json_node(buf, root);
+}
+
+
+
+gboolean bencode_collapse_json_item(bencode_item_t *item, JsonBuilder *builder);
+
+gboolean bencode_collapse_json_list(bencode_item_t *item, JsonBuilder *builder) {
+	json_builder_begin_array(builder);
+	for (bencode_item_t *el = item->child; el; el = el->sibling) {
+		if (!bencode_collapse_json_item(el, builder))
+			return FALSE;
+	}
+	json_builder_end_array(builder);
+	return TRUE;
+}
+
+gboolean bencode_collapse_json_string(bencode_item_t *item, JsonBuilder *builder) {
+	char buf[item->iov[1].iov_len + 1];
+	memcpy(buf, item->iov[1].iov_base, item->iov[1].iov_len);
+	buf[item->iov[1].iov_len] = '\0';
+	json_builder_add_string_value(builder, buf);
+	return TRUE;
+}
+
+gboolean bencode_collapse_json_dict(bencode_item_t *item, JsonBuilder *builder) {
+	json_builder_begin_object(builder);
+	bencode_item_t *val;
+	for (bencode_item_t *key = item->child; key; key = val->sibling) {
+		val = key->sibling;
+		if (key->type != BENCODE_STRING)
+			return FALSE;
+
+		char buf[key->iov[1].iov_len + 1];
+		memcpy(buf, key->iov[1].iov_base, key->iov[1].iov_len);
+		buf[key->iov[1].iov_len] = '\0';
+
+		json_builder_set_member_name(builder, buf);
+
+		if (!bencode_collapse_json_item(val, builder))
+			return FALSE;
+	}
+	json_builder_end_object(builder);
+	return TRUE;
+}
+
+gboolean bencode_collapse_json_int(bencode_item_t *item, JsonBuilder *builder) {
+	json_builder_add_int_value(builder, item->value);
+	return TRUE;
+}
+
+gboolean bencode_collapse_json_item(bencode_item_t *item, JsonBuilder *builder) {
+	switch (item->type) {
+		case BENCODE_LIST:
+			return bencode_collapse_json_list(item, builder);
+		case BENCODE_STRING:
+			return bencode_collapse_json_string(item, builder);
+		case BENCODE_DICTIONARY:
+			return bencode_collapse_json_dict(item, builder);
+		case BENCODE_INTEGER:
+			return bencode_collapse_json_int(item, builder);
+		default:
+			return FALSE;
+	}
+}
+
+str *bencode_collapse_str_json(bencode_item_t *root, str *out) {
+	JsonBuilder *builder = json_builder_new();
+	if (!bencode_collapse_json_item(root, builder))
+		goto err;
+	JsonGenerator *gen = json_generator_new();
+	JsonNode *json = json_builder_get_root(builder);
+	json_generator_set_root(gen, json);
+	char *result = json_generator_to_data(gen, NULL);
+	json_node_free(json);
+	g_object_unref(gen);
+	if (!result)
+		goto err;
+
+	out->s = result;
+	out->len = strlen(result);
+	bencode_buffer_destroy_add(root->buffer, free, result);
+	g_object_unref(builder);
+	return out;
+
+err:
+	g_object_unref(builder);
+	return NULL;
+}

daemon/control_ng.c

@@ -4,6 +4,7 @@
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <assert.h>
+#include <json-glib/json-glib.h>
 
 #include "obj.h"
 #include "poller.h"
@@ -157,6 +158,8 @@ int control_ng_process(str *buf, const endpoint_t *sin, char *addr,
 	resp = bencode_dictionary(&ngbuf->buffer);
 	assert(resp != NULL);
 
+	str *(*collapse_func)(bencode_item_t *root, str *out) = bencode_collapse_str;
+
 	cookie = *buf;
 	cookie.len -= data.len;
 	*data.s++ = '\0';
@@ -173,10 +176,28 @@ int control_ng_process(str *buf, const endpoint_t *sin, char *addr,
 		goto send_only;
 	}
 
-	dict = bencode_decode_expect_str(&ngbuf->buffer, &data, BENCODE_DICTIONARY);
-	errstr = "Could not decode dictionary";
-	if (!dict)
+	if (data.s[0] == 'd') {
+		dict = bencode_decode_expect_str(&ngbuf->buffer, &data, BENCODE_DICTIONARY);
+		errstr = "Could not decode bencode dictionary";
+		if (!dict)
+			goto err_send;
+	}
+	else if (data.s[0] == '{') {
+		collapse_func = bencode_collapse_str_json;
+		JsonParser *json = json_parser_new();
+		bencode_buffer_destroy_add(&ngbuf->buffer, g_object_unref, json);
+		errstr = "Failed to parse JSON document";
+		if (!json_parser_load_from_data(json, data.s, data.len, NULL))
+			goto err_send;
+		dict = bencode_convert_json(&ngbuf->buffer, json);
+		errstr = "Could not decode bencode dictionary";
+		if (!dict || dict->type != BENCODE_DICTIONARY)
+			goto err_send;
+	}
+	else {
+		errstr = "Invalid NG data format";
 		goto err_send;
+	}
 
 	bencode_dictionary_get_str(dict, "command", &cmd);
 	errstr = "Dictionary contains no key \"command\"";
@@ -344,7 +365,7 @@ err_send:
 	}
 
 send_resp:
-	bencode_collapse_str(resp, &reply);
+	collapse_func(resp, &reply);
 	to_send = &reply;
 
 	if (cmd.s) {

include/bencode.h

@@ -3,6 +3,7 @@
 
 #include <sys/uio.h>
 #include <string.h>
+#include <json-glib/json-glib.h>
 
 #include "compat.h"
 
@@ -222,7 +223,7 @@ struct iovec *bencode_iovec(bencode_item_t *root, int *cnt, unsigned int head, u
 char *bencode_collapse(bencode_item_t *root, size_t *len);
 
 /* Identical to bencode_collapse() but fills in a "str" object. Returns "out". */
-static str *bencode_collapse_str(bencode_item_t *root, str *out);
+INLINE str *bencode_collapse_str(bencode_item_t *root, str *out);
 
 /* Identical to bencode_collapse(), but the memory for the returned string is not allocated from
  * a bencode_buffer_t object, but instead using the function defined as BENCODE_MALLOC (normally
@@ -230,6 +231,9 @@ static str *bencode_collapse_str(bencode_item_t *root, str *out);
  * object can be destroyed, but the returned string remains valid and usable. */
 char *bencode_collapse_dup(bencode_item_t *root, size_t *len);
 
+// Collapse into a JSON document. Otherwise identical to bencode_collapse_str.
+str *bencode_collapse_str_json(bencode_item_t *root, str *out);
+
 
 
 
@@ -293,6 +297,9 @@ INLINE bencode_item_t *bencode_decode_expect_str(bencode_buffer_t *buf, const st
 /* Returns the number of bytes that could successfully be decoded from 's', -1 if more bytes are needed or -2 on error */
 ssize_t bencode_valid(const char *s, size_t len);
 
+// Convert a GLib JSON document to bencode
+bencode_item_t *bencode_convert_json(bencode_buffer_t *buf, JsonParser *json);
+
 
 /*** DICTIONARY LOOKUP & EXTRACTION ***/
 

t/Makefile

@@ -12,6 +12,7 @@ CFLAGS+=	$(shell pkg-config --cflags openssl)
 CFLAGS+=	-I. -I../lib/ -I../kernel-module/ -I../include/
 CFLAGS+=	-D_GNU_SOURCE
 CFLAGS+=	$(shell pkg-config --cflags libpcre)
+CFLAGS+=	$(shell pkg-config --cflags json-glib-1.0)
 ifeq ($(with_transcoding),yes)
 CFLAGS+=	$(shell pkg-config --cflags libavcodec)
 CFLAGS+=	$(shell pkg-config --cflags libavformat)
@@ -21,7 +22,6 @@ CFLAGS+=	$(shell pkg-config --cflags libavfilter)
 CFLAGS+=	$(shell pkg-config --cflags spandsp)
 CFLAGS+=	-DWITH_TRANSCODING
 CFLAGS+=	$(shell pkg-config --cflags zlib)
-CFLAGS+=	$(shell pkg-config --cflags json-glib-1.0)
 CFLAGS+=	$(shell pkg-config --cflags libwebsockets)
 CFLAGS+=	$(shell pkg-config --cflags libevent_pthreads)
 CFLAGS+=	$(shell pkg-config xmlrpc_client --cflags 2> /dev/null || xmlrpc-c-config client --cflags)
@@ -41,6 +41,7 @@ LDLIBS+=	$(shell pkg-config --libs gthread-2.0)
 LDLIBS+=	$(shell pkg-config --libs libcrypto)
 LDLIBS+=	$(shell pkg-config --libs openssl)
 LDLIBS+=	$(shell pkg-config --libs libpcre)
+LDLIBS+=	$(shell pkg-config --libs json-glib-1.0)
 ifeq ($(with_transcoding),yes)
 LDLIBS+=	$(shell pkg-config --libs libavcodec)
 LDLIBS+=	$(shell pkg-config --libs libavformat)
@@ -49,7 +50,6 @@ LDLIBS+=	$(shell pkg-config --libs libswresample)
 LDLIBS+=	$(shell pkg-config --libs libavfilter)
 LDLIBS+=	$(shell pkg-config --libs spandsp)
 LDLIBS+=	$(shell pkg-config --libs zlib)
-LDLIBS+=	$(shell pkg-config --libs json-glib-1.0)
 LDLIBS+=	$(shell pkg-config --libs libwebsockets)
 LDLIBS+=	-lpcap
 LDLIBS+=	$(shell pkg-config --libs libevent_pthreads)