TT#31404 support custom ptime and repacketization for sampled codecs

Change-Id: I4db83cb8faa9b0234dd655f4bdc116a3ddb59f39

README.md

@@ -46,7 +46,6 @@ the following additional features are available:
 
 *Rtpengine* does not (yet) support:
 
-* Repacketization or transcoding
 * Playback of pre-recorded streams/announcements
 * ZRTP, although ZRTP passes through *rtpengine* just fine
 
@@ -1259,6 +1258,12 @@ Optionally included keys are:
 		list of offered codecs, then no transcoding will be done. Also note that if
 		transcoding takes place, in-kernel forwarding is disabled for this media stream
 		and all processing happens in userspace.
+* `ptime`
+
+	Contains an integer. If set, changes the `a=ptime` attribute's value in the outgoing
+	SDP to the provided value. It also engages the transcoding engine for supported codecs
+	to provide repacketization functionality, even if no additional codec has actually
+	been requested for transcoding.
 
 
 An example of a complete `offer` request dictionary could be (SDP body abbreviated):

daemon/bencode.h

@@ -355,6 +355,9 @@ INLINE char *bencode_dictionary_get_str_dup(bencode_item_t *dict, const char *ke
  * specified which value should be returned if the key is not found or if the value is not an integer. */
 INLINE long long int bencode_dictionary_get_integer(bencode_item_t *dict, const char *key, long long int defval);
 
+/* Identical to bencode_dictionary_get_integer() but allows for the item to be a string. */
+INLINE long long int bencode_dictionary_get_int_str(bencode_item_t *dict, const char *key, long long int defval);
+
 /* Identical to bencode_dictionary_get(), but returns the object only if its type matches "expect". */
 INLINE bencode_item_t *bencode_dictionary_get_expect(bencode_item_t *dict, const char *key, bencode_type_t expect);
 
@@ -516,6 +519,30 @@ INLINE long long int bencode_dictionary_get_integer(bencode_item_t *dict, const
 	return val->value;
 }
 
+INLINE long long int bencode_dictionary_get_int_str(bencode_item_t *dict, const char *key, long long int defval) {
+	bencode_item_t *val;
+	val = bencode_dictionary_get(dict, key);
+	if (!val)
+		return defval;
+	if (val->type == BENCODE_INTEGER)
+		return val->value;
+	if (val->type != BENCODE_STRING)
+		return defval;
+	if (val->iov[1].iov_len == 0)
+		return defval;
+
+	// uh oh...
+	char *s = val->iov[1].iov_base;
+	char old = s[val->iov[1].iov_len];
+	s[val->iov[1].iov_len] = 0;
+	char *errp;
+	long long int ret = strtoll(val->iov[1].iov_base, &errp, 10);
+	s[val->iov[1].iov_len] = old;
+	if (errp != val->iov[1].iov_base + val->iov[1].iov_len)
+		return defval;
+	return ret;
+}
+
 INLINE bencode_item_t *bencode_decode_expect(bencode_buffer_t *buf, const char *s, int len, bencode_type_t expect) {
 	bencode_item_t *ret;
 	ret = bencode_decode(buf, s, len);

daemon/call.c

@@ -684,6 +684,7 @@ static struct call_media *__get_media(struct call_monologue *ml, GList **it, con
 	med->index = sp->index;
 	call_str_cpy(ml->call, &med->type, &sp->type);
 	med->codecs_recv = g_hash_table_new_full(g_int_hash, g_int_equal, NULL, NULL);
+	med->codecs_send = g_hash_table_new_full(g_int_hash, g_int_equal, NULL, NULL);
 	med->codec_names_recv = g_hash_table_new_full(str_hash, str_equal, NULL, (void (*)(void*)) g_queue_free);
 	med->codec_names_send = g_hash_table_new_full(str_hash, str_equal, NULL, (void (*)(void*)) g_queue_free);
 
@@ -1573,6 +1574,12 @@ int monologue_offer_answer(struct call_monologue *other_ml, GQueue *streams,
 		}
 
 		// codec and RTP payload types handling
+		if (sp->ptime > 0) {
+			media->ptime = sp->ptime;
+			other_media->ptime = sp->ptime;
+		}
+		if (flags->ptime > 0)
+			media->ptime = flags->ptime;
 		codec_rtp_payload_types(media, other_media, &sp->rtp_payload_types,
 				flags->codec_strip, &flags->codec_offer, &flags->codec_transcode);
 		codec_handlers_update(media, other_media);
@@ -1985,6 +1992,7 @@ static void __call_free(void *p) {
 		g_queue_clear(&md->streams);
 		g_queue_clear(&md->endpoint_maps);
 		g_hash_table_destroy(md->codecs_recv);
+		g_hash_table_destroy(md->codecs_send);
 		g_hash_table_destroy(md->codec_names_recv);
 		g_hash_table_destroy(md->codec_names_send);
 		g_queue_clear_full(&md->codecs_prefs_recv, (GDestroyNotify) payload_type_free);

daemon/call.h

@@ -240,6 +240,7 @@ struct stream_params {
 	GQueue			ice_candidates; /* slice-alloc'd */
 	str			ice_ufrag;
 	str			ice_pwd;
+	int			ptime;
 };
 
 struct endpoint_map {
@@ -332,6 +333,7 @@ struct call_media {
 	GQueue			codecs_prefs_recv; // preference by order in SDP; storage container
 
 	// what we can send, taken from received SDP:
+	GHashTable		*codecs_send; // int payload type -> struct rtp_payload_type
 	GHashTable		*codec_names_send; // codec name -> GQueue of int payload types; storage container
 	GQueue			codecs_prefs_send; // storage container
 
@@ -339,6 +341,8 @@ struct call_media {
 						// XXX combine this with 'codecs' hash table?
 	volatile struct codec_handler *codec_handler_cache;
 
+	int			ptime; // either from SDP or overridden
+
 	volatile unsigned int	media_flags;
 };
 

daemon/call_interfaces.c

@@ -690,9 +690,10 @@ static void call_ng_process_flags(struct sdp_ng_flags *out, bencode_item_t *inpu
 	bencode_get_alt(input, "media-address", "media address", &out->media_address);
 	if (bencode_get_alt(input, "address-family", "address family", &out->address_family_str))
 		out->address_family = get_socket_family_rfc(&out->address_family_str);
-	out->tos = bencode_dictionary_get_integer(input, "TOS", 256);
+	out->tos = bencode_dictionary_get_int_str(input, "TOS", 256);
 	bencode_get_alt(input, "record-call", "record call", &out->record_call_str);
 	bencode_dictionary_get_str(input, "metadata", &out->metadata);
+	out->ptime = bencode_dictionary_get_int_str(input, "ptime", 0);
 
 	if ((dict = bencode_dictionary_get_expect(input, "codec", BENCODE_DICTIONARY))) {
 		/* XXX module still needs to support these */
@@ -911,9 +912,9 @@ const char *call_delete_ng(bencode_item_t *input, bencode_item_t *output) {
 				fatal = 1;
 		}
 	}
-	delete_delay = bencode_dictionary_get_integer(input, "delete-delay", -1);
+	delete_delay = bencode_dictionary_get_int_str(input, "delete-delay", -1);
 	if (delete_delay == -1) {
-		delete_delay = bencode_dictionary_get_integer(input, "delete delay", -1);
+		delete_delay = bencode_dictionary_get_int_str(input, "delete delay", -1);
 		if (delete_delay == -1) {
 			/* legacy support */
 			str s;
@@ -1237,7 +1238,7 @@ const char *call_list_ng(bencode_item_t *input, bencode_item_t *output) {
 	bencode_item_t *calls = NULL;
 	long long int limit;
 
-	limit = bencode_dictionary_get_integer(input, "limit", 32);
+	limit = bencode_dictionary_get_int_str(input, "limit", 32);
 
 	if (limit < 0) {
 		return "invalid limit, must be >= 0";

daemon/call_interfaces.h

@@ -34,6 +34,7 @@ struct sdp_ng_flags {
 	GHashTable *codec_strip;
 	GQueue codec_offer;
 	GQueue codec_transcode;
+	int ptime;
 	int asymmetric:1,
 	    no_redis_update:1,
 	    unidirectional:1,

daemon/codec.c

@@ -18,9 +18,12 @@ struct codec_ssrc_handler {
 	decoder_t *decoder;
 	encoder_t *encoder;
 	format_t encoder_format;
-	unsigned long ts_offset;
+	int ptime;
+	int bytes_per_packet;
+	unsigned long ts_out;
 	u_int32_t ssrc_out;
 	u_int16_t seq_out;
+	GString *sample_buffer;
 };
 struct transcode_packet {
 	seq_packet_t p; // must be first
@@ -59,9 +62,9 @@ static void __codec_handler_free(void *pp) {
 	g_slice_free1(sizeof(*h), h);
 }
 
-static struct codec_handler *__handler_new(int pt) {
+static struct codec_handler *__handler_new(struct rtp_payload_type *pt) {
 	struct codec_handler *handler = g_slice_alloc0(sizeof(*handler));
-	handler->source_pt.payload_type = pt;
+	handler->source_pt = *pt;
 	return handler;
 }
 
@@ -142,9 +145,17 @@ void codec_handlers_update(struct call_media *receiver, struct call_media *sink)
 		__ensure_codec_def(pt);
 		if (!pt->codec_def || pt->codec_def->avcodec_id == -1) // not supported, next
 			continue;
-		ilog(LOG_DEBUG, "Default sink codec is " STR_FORMAT, STR_FMT(&pt->encoding));
-		pref_dest_codec = pt;
-		break;
+
+		// fix up ptime
+		if (!pt->ptime)
+			pt->ptime = pt->codec_def->default_ptime;
+		if (sink->ptime)
+			pt->ptime = sink->ptime;
+
+		if (!pref_dest_codec) {
+			ilog(LOG_DEBUG, "Default sink codec is " STR_FORMAT, STR_FMT(&pt->encoding));
+			pref_dest_codec = pt;
+		}
 	}
 
 	if (MEDIA_ISSET(sink, TRANSCODE)) {
@@ -219,7 +230,7 @@ void codec_handlers_update(struct call_media *receiver, struct call_media *sink)
 		handler = g_hash_table_lookup(receiver->codec_handlers, &pt->payload_type);
 		if (!handler) {
 			ilog(LOG_DEBUG, "Creating codec handler for " STR_FORMAT, STR_FMT(&pt->encoding));
-			handler = __handler_new(pt->payload_type);
+			handler = __handler_new(pt);
 			g_hash_table_insert(receiver->codec_handlers, &handler->source_pt.payload_type,
 					handler);
 		}
@@ -233,6 +244,12 @@ void codec_handlers_update(struct call_media *receiver, struct call_media *sink)
 			goto next;
 		}
 
+		// figure out our ptime
+		if (!pt->ptime)
+			pt->ptime = pt->codec_def->default_ptime;
+		if (receiver->ptime)
+			pt->ptime = receiver->ptime;
+
 		// if the sink's codec preferences are unknown (empty), or there are
 		// no supported codecs to transcode to, then we have nothing
 		// to do. most likely this is an initial offer without a received answer.
@@ -243,18 +260,49 @@ void codec_handlers_update(struct call_media *receiver, struct call_media *sink)
 			goto next;
 		}
 
-		if (g_hash_table_lookup(sink->codec_names_send, &pt->encoding)) {
-			// the sink supports this codec. forward without transcoding.
+		struct rtp_payload_type *dest_pt; // transcode to this
+
+		// in case of ptime mismatch, we transcode
+		//struct rtp_payload_type *dest_pt = g_hash_table_lookup(sink->codec_names_send, &pt->encoding);
+		GQueue *dest_codecs = g_hash_table_lookup(sink->codec_names_send, &pt->encoding);
+		if (dest_codecs) {
+			// the sink supports this codec - check offered formats
+			dest_pt = NULL;
+			for (GList *k = dest_codecs->head; k; k = k->next) {
+				unsigned int dest_ptype = GPOINTER_TO_UINT(k->data);
+				dest_pt = g_hash_table_lookup(sink->codecs_send, &dest_ptype);
+				if (!dest_pt)
+					continue;
+				// XXX match up format parameters
+				break;
+			}
+
+			if (!dest_pt)
+				goto unsupported;
+
+			// in case of ptime mismatch, we transcode, but between the same codecs
+			if (dest_pt->ptime && pt->ptime
+					&& dest_pt->ptime != pt->ptime)
+			{
+				ilog(LOG_DEBUG, "Mismatched ptime between source and sink (%i <> %i), "
+						"enabling transcoding",
+					dest_pt->ptime, pt->ptime);
+				goto transcode;
+			}
+
 			// XXX check format parameters as well
 			ilog(LOG_DEBUG, "Sink supports codec " STR_FORMAT, STR_FMT(&pt->encoding));
 			__make_passthrough(handler);
 			goto next;
 		}
 
+unsupported:
 		// the sink does not support this codec -> transcode
 		ilog(LOG_DEBUG, "Sink does not support codec " STR_FORMAT, STR_FMT(&pt->encoding));
+		dest_pt = pref_dest_codec;
+transcode:
 		MEDIA_SET(receiver, TRANSCODE);
-		__make_transcoder(handler, pt, pref_dest_codec);
+		__make_transcoder(handler, pt, dest_pt);
 
 next:
 		l = l->next;
@@ -344,7 +392,9 @@ static struct ssrc_entry *__ssrc_handler_new(void *p) {
 	packet_sequencer_init(&ch->sequencer, (GDestroyNotify) __transcode_packet_free);
 	ch->seq_out = random();
 	ch->ssrc_out = ssrc_out;
-	ch->ts_offset = random();
+	ch->ts_out = random();
+	ch->ptime = h->dest_pt.ptime;
+	ch->sample_buffer = g_string_new("");
 
 	format_t enc_format = {
 		.clockrate = h->dest_pt.clock_rate * h->dest_pt.codec_def->clockrate_mult,
@@ -356,16 +406,23 @@ static struct ssrc_entry *__ssrc_handler_new(void *p) {
 		goto err;
 	// XXX make bitrate configurable
 	if (encoder_config(ch->encoder, h->dest_pt.codec_def->avcodec_id,
-				h->dest_pt.codec_def->default_bitrate, &enc_format, &ch->encoder_format))
+				h->dest_pt.codec_def->default_bitrate,
+				ch->ptime / h->dest_pt.codec_def->clockrate_mult,
+				&enc_format, &ch->encoder_format))
 		goto err;
 
-	ilog(LOG_DEBUG, "Encoder created with clockrate %i, %i channels, using sample format %i",
-			ch->encoder_format.clockrate, ch->encoder_format.channels, ch->encoder_format.format);
-
 	ch->decoder = decoder_new_fmt(h->source_pt.codec_def, h->source_pt.clock_rate, h->source_pt.channels,
 			&ch->encoder_format);
 	if (!ch->decoder)
 		goto err;
+
+	ch->bytes_per_packet = ch->encoder->samples_per_frame * h->dest_pt.codec_def->bits_per_sample / 8;
+
+	ilog(LOG_DEBUG, "Encoder created with clockrate %i, %i channels, using sample format %i "
+			"(ptime %i for %i samples or %i bytes per packet)",
+			ch->encoder_format.clockrate, ch->encoder_format.channels, ch->encoder_format.format,
+			ch->ptime, ch->encoder->samples_per_frame, ch->bytes_per_packet);
+
 	return &ch->h;
 
 err:
@@ -390,6 +447,7 @@ static void __ssrc_handler_free(struct codec_ssrc_handler *ch) {
 		} while (going);
 		encoder_free(ch->encoder);
 	}
+	g_string_free(ch->sample_buffer, TRUE);
 	g_slice_free1(sizeof(*ch), ch);
 }
 
@@ -400,26 +458,56 @@ static int __packet_encoded(encoder_t *enc, void *u1, void *u2) {
 	ilog(LOG_DEBUG, "RTP media successfully encoded: TS %llu, len %i",
 			(unsigned long long) enc->avpkt.pts, enc->avpkt.size);
 
-	// reconstruct RTP header
-	unsigned int pkt_len = enc->avpkt.size + sizeof(struct rtp_header);
-	char *buf = malloc(pkt_len);
-	struct rtp_header *rh = (void *) buf;
+	// run this through our packetizer
+	AVPacket *in_pkt = &enc->avpkt;
 
-	ZERO(*rh);
-	rh->v_p_x_cc = 0x80;
-	rh->m_pt = ch->handler->dest_pt.payload_type;
-	rh->seq_num = htons(ch->seq_out++);
-	rh->timestamp = htonl(enc->avpkt.pts + ch->ts_offset);
-	rh->ssrc = ch->ssrc_out;
+	while (1) {
+		// figure out how big of a buffer we need
+		unsigned int payload_len = MAX(enc->avpkt.size, ch->bytes_per_packet);
+		unsigned int pkt_len = sizeof(struct rtp_header) + payload_len;
+		// prepare our buffers
+		char *buf = malloc(pkt_len);
+		struct rtp_header *rh = (void *) buf;
+		char *payload = buf + sizeof(struct rtp_header);
+		// tell our packetizer how much we want
+		str inout;
+		str_init_len(&inout, payload, payload_len);
+		// and request a packet
+		if (in_pkt)
+			ilog(LOG_DEBUG, "Adding %i bytes to packetizer", in_pkt->size);
+		int ret = ch->handler->dest_pt.codec_def->packetizer(in_pkt,
+				ch->sample_buffer, &inout);
+
+		if (G_UNLIKELY(ret == -1)) {
+			// nothing
+			free(buf);
+			break;
+		}
 
-	// XXX use writev() for output? would make sense if enc->avpkt.data can be stolen
-	memcpy(buf + sizeof(struct rtp_header), enc->avpkt.data, enc->avpkt.size);
+		ilog(LOG_DEBUG, "Received packet of %i bytes from packetizer", inout.len);
+		// reconstruct RTP header
+		ZERO(*rh);
+		rh->v_p_x_cc = 0x80;
+		rh->m_pt = ch->handler->dest_pt.payload_type;
+		rh->seq_num = htons(ch->seq_out++);
+		rh->timestamp = htonl(enc->avpkt.pts + ch->ts_out);
+		rh->ssrc = ch->ssrc_out;
+
+		// add to output queue
+		struct codec_packet *p = g_slice_alloc(sizeof(*p));
+		p->s.s = buf;
+		p->s.len = inout.len + sizeof(struct rtp_header);
+		p->free_func = free;
+		g_queue_push_tail(out_q, p);
+
+		if (ret == 0) {
+			// no more to go
+			break;
+		}
 
-	struct codec_packet *p = g_slice_alloc(sizeof(*p));
-	p->s.s = buf;
-	p->s.len = pkt_len;
-	p->free_func = free;
-	g_queue_push_tail(out_q, p);
+		// loop around and get more
+		in_pkt = NULL;
+	}
 
 	return 0;
 }
@@ -430,7 +518,7 @@ static int __packet_decoded(decoder_t *decoder, AVFrame *frame, void *u1, void *
 	ilog(LOG_DEBUG, "RTP media successfully decoded: TS %llu, samples %u",
 			(unsigned long long) frame->pts, frame->nb_samples);
 
-	encoder_input_data(ch->encoder, frame, __packet_encoded, ch, u2);
+	encoder_input_fifo(ch->encoder, frame, __packet_encoded, ch, u2);
 
 	av_frame_free(&frame);
 	return 0;
@@ -608,6 +696,7 @@ static void __rtp_payload_type_add_recv(struct call_media *media,
 // duplicates 'pt'
 static void __rtp_payload_type_add_send(struct call_media *other_media, struct rtp_payload_type *pt) {
 	pt = __rtp_payload_type_copy(pt);
+	g_hash_table_insert(other_media->codecs_send, &pt->payload_type, pt);
 	__rtp_payload_type_add_name(other_media->codec_names_send, pt);
 	g_queue_push_tail(&other_media->codecs_prefs_send, pt);
 }
@@ -656,6 +745,7 @@ void codec_rtp_payload_types(struct call_media *media, struct call_media *other_
 	g_hash_table_remove_all(media->codec_names_recv);
 	// and sending part for 'other_media'
 	g_queue_clear_full(&other_media->codecs_prefs_send, (GDestroyNotify) payload_type_free);
+	g_hash_table_remove_all(other_media->codecs_send);
 	g_hash_table_remove_all(other_media->codec_names_send);
 
 	if (strip && g_hash_table_lookup(strip, &str_all))

daemon/sdp.c

@@ -202,6 +202,7 @@ struct sdp_attribute {
 		ATTR_FMTP,
 		ATTR_IGNORE,
 		ATTR_RTPENGINE,
+		ATTR_PTIME,
 		ATTR_END_OF_CANDIDATES,
 	} attr;
 
@@ -834,6 +835,8 @@ static int parse_attribute(struct sdp_attribute *a) {
 				ret = parse_attribute_group(a);
 			else if (!str_cmp(&a->name, "setup"))
 				ret = parse_attribute_setup(a);
+			else if (!str_cmp(&a->name, "ptime"))
+				a->attr = ATTR_PTIME;
 			break;
 		case 6:
 			if (!str_cmp(&a->name, "crypto"))
@@ -1125,6 +1128,7 @@ static int __rtp_payload_types(struct stream_params *sp, struct sdp_media *media
 	GList *ql;
 	struct sdp_attribute *attr;
 	int ret = 0;
+	int ptime = 0;
 
 	if (!sp->protocol || !sp->protocol->rtp)
 		return 0;
@@ -1146,13 +1150,18 @@ static int __rtp_payload_types(struct stream_params *sp, struct sdp_media *media
 		g_hash_table_insert(ht_fmtp, &attr->u.fmtp.payload_type, &attr->u.fmtp.format_parms_str);
 	}
 
+	// check for a=ptime
+	attr = attr_get_by_id(&media->attributes, ATTR_PTIME);
+	if (attr && attr->value.s)
+		ptime = atoi(attr->value.s);
+
 	/* then go through the format list and associate */
 	for (ql = media->format_list.head; ql; ql = ql->next) {
 		char *ep;
 		str *s;
 		unsigned int i;
 		struct rtp_payload_type *pt;
-		const struct rtp_payload_type *ptl;
+		const struct rtp_payload_type *ptl, *ptrfc;
 
 		s = ql->data;
 		i = (unsigned int) strtoul(s->s, &ep, 10);
@@ -1161,11 +1170,14 @@ static int __rtp_payload_types(struct stream_params *sp, struct sdp_media *media
 
 		/* first look in rtpmap for a match, then check RFC types,
 		 * else fall back to an "unknown" type */
-		ptl = rtp_payload_type(i, ht_rtpmap);
+		ptrfc = rtp_get_rfc_payload_type(i);
+		ptl = g_hash_table_lookup(ht_rtpmap, &i);
 
 		pt = g_slice_alloc0(sizeof(*pt));
 		if (ptl)
 			*pt = *ptl;
+		else if (ptrfc)
+			*pt = *ptrfc;
 		else
 			pt->payload_type = i;
 
@@ -1173,6 +1185,12 @@ static int __rtp_payload_types(struct stream_params *sp, struct sdp_media *media
 		if (s)
 			pt->format_parameters = *s;
 
+		// fill in ptime
+		if (ptime)
+			pt->ptime = ptime;
+		else if (!pt->ptime && ptrfc)
+			pt->ptime = ptrfc->ptime;
+
 		g_queue_push_tail(&sp->rtp_payload_types, pt);
 	}
 
@@ -1325,6 +1343,11 @@ int sdp_streams(const GQueue *sessions, GQueue *streams, struct sdp_ng_flags *fl
 
 			__sdp_ice(sp, media);
 
+			// a=ptime
+			attr = attr_get_by_id(&media->attributes, ATTR_PTIME);
+			if (attr && attr->value.s)
+				sp->ptime = atoi(attr->value.s);
+
 			/* determine RTCP endpoint */
 
 			if (attr_get_by_id(&media->attributes, ATTR_RTCP_MUX)) {
@@ -1694,6 +1717,7 @@ static int process_media_attributes(struct sdp_chopper *chop, struct sdp_media *
 			case ATTR_END_OF_CANDIDATES: // we strip it here and re-insert it later
 			case ATTR_RTPMAP:
 			case ATTR_FMTP:
+			case ATTR_PTIME:
 				goto strip;
 
 			case ATTR_EXTMAP:
@@ -2103,6 +2127,9 @@ int sdp_replace(struct sdp_chopper *chop, GQueue *sessions, struct call_monologu
 			insert_crypto(call_media, chop);
 			insert_dtls(call_media, chop);
 
+			if (call_media->ptime)
+				chopper_append_printf(chop, "a=ptime:%i\r\n", call_media->ptime);
+
 			if (MEDIA_ISSET(call_media, ICE) && call_media->ice_agent) {
 				chopper_append_c(chop, "a=ice-ufrag:");
 				chopper_append_str(chop, &call_media->ice_agent->ufrag[1]);

lib/codeclib.c

@@ -26,7 +26,13 @@
 
 
 
-#define CODEC_DEF_FULL(ref, codec_id, mult, name, clockrate, channels, bitrate) { \
+packetizer_f packetizer_passthrough; // pass frames as they arrive in AVPackets
+packetizer_f packetizer_samplestream; // flat stream of samples
+
+
+
+
+#define CODEC_DEF_FULL(ref, codec_id, mult, name, clockrate, channels, bitrate, ptime, pizer, bps) { \
 	.rtpname = #ref, \
 	.avcodec_id = codec_id, \
 	.clockrate_mult = mult, \
@@ -34,45 +40,53 @@
 	.default_clockrate = clockrate, \
 	.default_channels = channels, \
 	.default_bitrate = bitrate, \
+	.default_ptime = ptime, \
+	.packetizer = packetizer_ ## pizer, \
+	.bits_per_sample = bps, \
 }
-#define CODEC_DEF_AVC(ref, id, mult, name, clockrate, channels, bitrate) \
-	CODEC_DEF_FULL(ref, AV_CODEC_ID_ ## id, mult, name, clockrate, channels, bitrate)
-#define CODEC_DEF_MULT_NAME(ref, id, mult, name) CODEC_DEF_AVC(ref, id, mult, name, -1, -1, 0)
-#define CODEC_DEF_MULT_NAME_ENC(ref, id, mult, name, clockrate, channels, bitrate) \
-	CODEC_DEF_AVC(ref, id, mult, name, clockrate, channels, bitrate)
-#define CODEC_DEF_MULT(ref, id, mult) CODEC_DEF_MULT_NAME(ref, id, mult, NULL)
-#define CODEC_DEF_MULT_ENC(ref, id, mult, clockrate, channels) \
-	CODEC_DEF_MULT_NAME_ENC(ref, id, mult, NULL, clockrate, channels, 0)
-#define CODEC_DEF_NAME(ref, id, name) CODEC_DEF_MULT_NAME(ref, id, 1, name)
-#define CODEC_DEF_NAME_ENC(ref, id, name, clockrate, channels, bitrate) \
-	CODEC_DEF_MULT_NAME_ENC(ref, id, 1, name, clockrate, channels, bitrate)
-#define CODEC_DEF(ref, id) CODEC_DEF_MULT(ref, id, 1)
-#define CODEC_DEF_ENC(ref, id, clockrate, channels) CODEC_DEF_MULT_ENC(ref, id, 1, clockrate, channels)
-#define CODEC_DEF_STUB(ref) CODEC_DEF_FULL(ref, -1, 1, ref, -1, -1, 0)
+#define CODEC_DEF_AVC(ref, id, mult, name, clockrate, channels, bitrate, ptime, pizer, bps) \
+	CODEC_DEF_FULL(ref, AV_CODEC_ID_ ## id, mult, name, clockrate, channels, bitrate, ptime, pizer, bps)
+#define CODEC_DEF_MULT_NAME(ref, id, mult, name, pizer, bps) CODEC_DEF_AVC(ref, id, mult, name, -1, -1, 0, 0, pizer, bps)
+#define CODEC_DEF_MULT(ref, id, mult, pizer, bps) CODEC_DEF_MULT_NAME(ref, id, mult, NULL, pizer, bps)
+#define CODEC_DEF_NAME(ref, id, name, pizer, bps) CODEC_DEF_MULT_NAME(ref, id, 1, name, pizer, bps)
+#define CODEC_DEF(ref, id, pizer, bps) CODEC_DEF_MULT(ref, id, 1, pizer, bps)
+
+// _ENC macros provided for codecs not having defaults in the RTP RFC
+#define CODEC_DEF_NAME_ENC(ref, id, name, clockrate, channels, bitrate, ptime, pizer, bps) \
+	CODEC_DEF_MULT_NAME_ENC(ref, id, 1, name, clockrate, channels, bitrate, ptime, pizer, bps)
+#define CODEC_DEF_MULT_ENC(ref, id, mult, clockrate, channels, bitrate, ptime, pizer, bps) \
+	CODEC_DEF_MULT_NAME_ENC(ref, id, mult, NULL, clockrate, channels, bitrate, ptime, pizer, bps)
+#define CODEC_DEF_ENC(ref, id, clockrate, channels, bitrate, ptime, pizer, bps) \
+	CODEC_DEF_MULT_ENC(ref, id, 1, clockrate, channels, bitrate, ptime, pizer, bps)
+#define CODEC_DEF_MULT_NAME_ENC(ref, id, mult, name, clockrate, channels, bitrate, ptime, pizer, bps) \
+	CODEC_DEF_AVC(ref, id, mult, name, clockrate, channels, bitrate, ptime, pizer, bps)
+
+// not real audio codecs
+#define CODEC_DEF_STUB(ref) CODEC_DEF_FULL(ref, -1, 1, ref, -1, -1, 0, 0, passthrough, 0)
 
 static const struct codec_def_s codecs[] = {
-	CODEC_DEF(PCMA, PCM_ALAW),
-	CODEC_DEF(PCMU, PCM_MULAW),
-	CODEC_DEF(G723, G723_1),
-	CODEC_DEF_MULT(G722, ADPCM_G722, 2),
-	CODEC_DEF(QCELP, QCELP),
-	CODEC_DEF(G729, G729),
-	CODEC_DEF_ENC(speex, SPEEX, 16000, 1),
-	CODEC_DEF(GSM, GSM),
-	CODEC_DEF(iLBC, ILBC),
-	CODEC_DEF_NAME_ENC(opus, OPUS, libopus, 48000, 2, 24000),
-	CODEC_DEF_NAME(vorbis, VORBIS, libvorbis),
-	CODEC_DEF(ac3, AC3),
-	CODEC_DEF(eac3, EAC3),
-	CODEC_DEF(ATRAC3, ATRAC3),
-	CODEC_DEF(ATRAC-X, ATRAC3P),
+	CODEC_DEF(PCMA, PCM_ALAW, samplestream, 8),
+	CODEC_DEF(PCMU, PCM_MULAW, samplestream, 8),
+	CODEC_DEF(G723, G723_1, passthrough, 0),
+	CODEC_DEF_MULT(G722, ADPCM_G722, 2, samplestream, 8),
+	CODEC_DEF(QCELP, QCELP, passthrough, 0),
+	CODEC_DEF(G729, G729, passthrough, 0),
+	CODEC_DEF_ENC(speex, SPEEX, 16000, 1, 11000, 20, passthrough, 0),
+	CODEC_DEF(GSM, GSM, passthrough, 0),
+	CODEC_DEF(iLBC, ILBC, passthrough, 0),
+	CODEC_DEF_NAME_ENC(opus, OPUS, libopus, 48000, 2, 32000, 20, passthrough, 0),
+	CODEC_DEF_NAME(vorbis, VORBIS, libvorbis, passthrough, 0),
+	CODEC_DEF(ac3, AC3, passthrough, 0),
+	CODEC_DEF(eac3, EAC3, passthrough, 0),
+	CODEC_DEF(ATRAC3, ATRAC3, passthrough, 0),
+	CODEC_DEF(ATRAC-X, ATRAC3P, passthrough, 0),
 #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 0, 0)
-	CODEC_DEF(EVRC, EVRC),
-	CODEC_DEF(EVRC0, EVRC),
-	CODEC_DEF(EVRC1, EVRC),
+	CODEC_DEF(EVRC, EVRC, passthrough, 0),
+	CODEC_DEF(EVRC0, EVRC, passthrough, 0),
+	CODEC_DEF(EVRC1, EVRC, passthrough, 0),
 #endif
-	CODEC_DEF_ENC(AMR, AMR_NB, 8000, 1),
-	CODEC_DEF_ENC(AMR-WB, AMR_WB, 16000, 1),
+	CODEC_DEF_ENC(AMR, AMR_NB, 8000, 1, 6600, 20, passthrough, 0),
+	CODEC_DEF_ENC(AMR-WB, AMR_WB, 16000, 1, 14250, 20, passthrough, 0),
 	CODEC_DEF_STUB(telephone-event),
 };
 
@@ -171,6 +185,8 @@ int decoder_input_data(decoder_t *dec, const str *data, unsigned long ts,
 
 	if (G_UNLIKELY(!dec))
 		return -1;
+	if (!data || !data->s || !data->len)
+		return 0;
 
 	dbg("%p dec pts %llu rtp_ts %llu incoming ts %lu", dec, (unsigned long long) dec->pts,
 			(unsigned long long) dec->rtp_ts, (unsigned long) ts);
@@ -477,8 +493,8 @@ encoder_t *encoder_new() {
 	return ret;
 }
 
-int encoder_config(encoder_t *enc, int codec_id, int bitrate, const format_t *requested_format,
-		format_t *actual_format)
+int encoder_config(encoder_t *enc, int codec_id, int bitrate, int ptime,
+		const format_t *requested_format, format_t *actual_format)
 {
 	const char *err;
 
@@ -526,6 +542,25 @@ int encoder_config(encoder_t *enc, int codec_id, int bitrate, const format_t *re
 
 	av_init_packet(&enc->avpkt);
 
+	enc->samples_per_frame = enc->actual_format.clockrate * ptime / 1000;
+
+// output frame and fifo
+	enc->frame = av_frame_alloc();
+	enc->frame->nb_samples = enc->avcctx->frame_size ? : (enc->samples_per_frame ? : 256);
+	enc->frame->format = enc->avcctx->sample_fmt;
+	enc->frame->sample_rate = enc->avcctx->sample_rate;
+	enc->frame->channel_layout = enc->avcctx->channel_layout;
+	if (!enc->frame->channel_layout)
+		enc->frame->channel_layout = av_get_default_channel_layout(enc->avcctx->channels);
+	if (av_frame_get_buffer(enc->frame, 0) < 0)
+		abort();
+
+	enc->fifo = av_audio_fifo_alloc(enc->avcctx->sample_fmt, enc->avcctx->channels,
+			enc->frame->nb_samples);
+
+	ilog(LOG_DEBUG, "Initialized encoder with frame size %u samples", enc->frame->nb_samples);
+
+
 done:
 	*actual_format = enc->actual_format;
 	return 0;
@@ -546,7 +581,11 @@ void encoder_close(encoder_t *enc) {
 	enc->avcctx = NULL;
 	format_init(&enc->requested_format);
 	format_init(&enc->actual_format);
+	av_audio_fifo_free(enc->fifo);
+	av_frame_free(&enc->frame);
 	enc->mux_dts = 0;
+	enc->fifo = NULL;
+	enc->fifo_pts = 0;
 }
 void encoder_free(encoder_t *enc) {
 	encoder_close(enc);
@@ -556,6 +595,11 @@ void encoder_free(encoder_t *enc) {
 int encoder_input_data(encoder_t *enc, AVFrame *frame,
 		int (*callback)(encoder_t *, void *u1, void *u2), void *u1, void *u2)
 {
+	if (G_UNLIKELY(!enc->avcctx))
+		return -1;
+	if (G_UNLIKELY(!enc->codec))
+		return -1;
+
 	int keep_going;
 	int have_frame = 1;
 	do {
@@ -632,3 +676,70 @@ int encoder_input_data(encoder_t *enc, AVFrame *frame,
 
 	return 0;
 }
+
+static int encoder_fifo_flush(encoder_t *enc,
+		int (*callback)(encoder_t *, void *u1, void *u2), void *u1, void *u2)
+{
+	while (av_audio_fifo_size(enc->fifo) >= enc->frame->nb_samples) {
+
+		if (av_audio_fifo_read(enc->fifo, (void **) enc->frame->data,
+					enc->frame->nb_samples) <= 0)
+			abort();
+
+		dbg("output fifo pts %lu",(unsigned long) enc->fifo_pts);
+		enc->frame->pts = enc->fifo_pts;
+
+		encoder_input_data(enc, enc->frame, callback, u1, u2);
+
+		enc->fifo_pts += enc->frame->nb_samples;
+	}
+
+	return 0;
+}
+
+int encoder_input_fifo(encoder_t *enc, AVFrame *frame,
+		int (*callback)(encoder_t *, void *u1, void *u2), void *u1, void *u2)
+{
+	// fix up output pts
+	if (av_audio_fifo_size(enc->fifo) == 0)
+		enc->fifo_pts = frame->pts;
+
+	if (av_audio_fifo_write(enc->fifo, (void **) frame->extended_data, frame->nb_samples) < 0)
+		return -1;
+
+	return encoder_fifo_flush(enc, callback, u1, u2);
+}
+
+
+int packetizer_passthrough(AVPacket *pkt, GString *buf, str *output) {
+	if (!pkt)
+		return -1;
+	assert(output->len >= pkt->size);
+	output->len = pkt->size;
+	memcpy(output->s, pkt->data, pkt->size);
+	return 0;
+}
+
+// returns: -1 = not enough data, nothing returned; 0 = returned a packet;
+// 1 = returned a packet and there's more
+int packetizer_samplestream(AVPacket *pkt, GString *buf, str *input_output) {
+	// avoid moving buffers around if possible:
+	// most common case: new input packet has just enough (or more) data as what we need
+	if (G_LIKELY(pkt && buf->len == 0 && pkt->size >= input_output->len)) {
+		memcpy(input_output->s, pkt->data, input_output->len);
+		if (pkt->size > input_output->len) // any leftovers?
+			g_string_append_len(buf, (char *) pkt->data + input_output->len,
+					pkt->size - input_output->len);
+		return buf->len >= input_output->len ? 1 : 0;
+	}
+	// we have to move data around. append input packet to buffer if we have one
+	if (pkt)
+		g_string_append_len(buf, (char *) pkt->data, pkt->size);
+	// do we have enough?
+	if (buf->len < input_output->len)
+		return -1;
+	// copy requested data into provided output buffer and remove from interim buffer
+	memcpy(input_output->s, buf->str, input_output->len);
+	g_string_erase(buf, 0, input_output->len);
+	return buf->len >= input_output->len ? 1 : 0;
+}

lib/codeclib.h

@@ -4,6 +4,7 @@
 
 #include <libavresample/avresample.h>
 #include <libavcodec/avcodec.h>
+#include <libavutil/audio_fifo.h>
 #include "str.h"
 
 
@@ -24,16 +25,21 @@ typedef struct resample_s resample_t;
 typedef struct seq_packet_s seq_packet_t;
 typedef struct packet_sequencer_s packet_sequencer_t;
 
+typedef int packetizer_f(AVPacket *, GString *, str *);
+
 
 
 struct codec_def_s {
-	const char *rtpname;
+	const char * const rtpname;
 	const int clockrate_mult;
 	const int avcodec_id;
 	const char *avcodec_name;
 	const int default_clockrate;
 	const int default_channels;
 	const int default_bitrate;
+	const int default_ptime;
+	packetizer_f * const packetizer;
+	const int bits_per_sample;
 };
 
 struct format_s {
@@ -68,6 +74,10 @@ struct encoder_s {
 	AVCodec *codec;
 	AVCodecContext *avcctx;
 	AVPacket avpkt;
+	AVAudioFifo *fifo;
+	int64_t fifo_pts; // pts of first data in fifo
+	int samples_per_frame;
+	AVFrame *frame; // to pull samples from the fifo
 	int64_t mux_dts; // last dts passed to muxer
 };
 
@@ -94,12 +104,14 @@ int decoder_input_data(decoder_t *dec, const str *data, unsigned long ts,
 
 
 encoder_t *encoder_new();
-int encoder_config(encoder_t *enc, int codec_id, int bitrate,
+int encoder_config(encoder_t *enc, int codec_id, int bitrate, int ptime,
 		const format_t *requested_format, format_t *actual_format);
 void encoder_close(encoder_t *);
 void encoder_free(encoder_t *);
 int encoder_input_data(encoder_t *enc, AVFrame *frame,
 		int (*callback)(encoder_t *, void *u1, void *u2), void *u1, void *u2);
+int encoder_input_fifo(encoder_t *enc, AVFrame *frame,
+		int (*callback)(encoder_t *, void *u1, void *u2), void *u1, void *u2);
 
 
 void packet_sequencer_init(packet_sequencer_t *ps, GDestroyNotify);

lib/rtplib.c

@@ -14,27 +14,29 @@ struct rtp_extension {
 
 
 
-#define RFC_TYPE(type, name, c_rate)					\
+#define RFC_TYPE_FULL(type, name, c_rate, chans, pt)			\
 	[type] = {							\
 		.payload_type		= type,				\
 		.encoding		= STR_CONST_INIT(#name),	\
 		.encoding_with_params	= STR_CONST_INIT(#name "/" #c_rate),	\
 		.clock_rate		= c_rate,			\
-		.channels		= 1,				\
+		.channels		= chans,			\
+		.ptime			= pt,				\
 	}
+#define RFC_TYPE(type, name, c_rate) RFC_TYPE_FULL(type, name, c_rate, 1, 20)
 
 const struct rtp_payload_type rfc_rtp_payload_types[] =
 {
 	RFC_TYPE(0, PCMU, 8000),
 	RFC_TYPE(3, GSM, 8000),
-	RFC_TYPE(4, G723, 8000),
+	RFC_TYPE_FULL(4, G723, 8000, 1, 30),
 	RFC_TYPE(5, DVI4, 8000),
 	RFC_TYPE(6, DVI4, 16000),
 	RFC_TYPE(7, LPC, 8000),
 	RFC_TYPE(8, PCMA, 8000),
 	RFC_TYPE(9, G722, 8000),
 	RFC_TYPE(10, L16, 44100),
-	RFC_TYPE(11, L16, 44100),
+	RFC_TYPE_FULL(11, L16, 44100, 2, 20),
 	RFC_TYPE(12, QCELP, 8000),
 	RFC_TYPE(13, CN, 8000),
 	RFC_TYPE(14, MPA, 90000),

lib/rtplib.h

@@ -25,6 +25,8 @@ struct rtp_payload_type {
 	int channels; // 2
 	str format_parameters; // value of a=fmtp
 
+	int ptime; // default from RFC
+
 	const codec_def_t *codec_def;
 };
 

recording-daemon/output.c

@@ -29,25 +29,6 @@ static int output_got_packet(encoder_t *enc, void *u1, void *u2) {
 
 	av_write_frame(output->fmtctx, &enc->avpkt);
 
-	output->fifo_pts += output->frame->nb_samples;
-
-	return 0;
-}
-
-
-static int output_flush(output_t *output) {
-	while (av_audio_fifo_size(output->fifo) >= output->frame->nb_samples) {
-
-		if (av_audio_fifo_read(output->fifo, (void **) output->frame->data,
-					output->frame->nb_samples) <= 0)
-			abort();
-
-		dbg("{%s} output fifo pts %lu", output->file_name, (unsigned long) output->fifo_pts);
-		output->frame->pts = output->fifo_pts;
-
-		encoder_input_data(output->encoder, output->frame, output_got_packet, output, NULL);
-	}
-
 	return 0;
 }
 
@@ -55,19 +36,9 @@ static int output_flush(output_t *output) {
 int output_add(output_t *output, AVFrame *frame) {
 	if (!output)
 		return -1;
-	if (!output->frame) // not ready - not configured
+	if (!output->encoder) // not ready - not configured
 		return -1;
-
-	dbg("{%s} output fifo size %u fifo_pts %lu", output->file_name, (unsigned int) av_audio_fifo_size(output->fifo),
-			(unsigned long) output->fifo_pts);
-	// fix up output pts
-	if (av_audio_fifo_size(output->fifo) == 0)
-		output->fifo_pts = frame->pts;
-
-	if (av_audio_fifo_write(output->fifo, (void **) frame->extended_data, frame->nb_samples) < 0)
-		return -1;
-
-	return output_flush(output);
+	return encoder_input_fifo(output->encoder, frame, output_got_packet, output, NULL);
 }
 
 
@@ -102,7 +73,7 @@ int output_config(output_t *output, const format_t *requested_format, format_t *
 	if (!output->fmtctx->oformat)
 		goto err;
 
-	if (encoder_config(output->encoder, output_codec_id, mp3_bitrate, requested_format, actual_format))
+	if (encoder_config(output->encoder, output_codec_id, mp3_bitrate, 0, requested_format, actual_format))
 		goto err;
 
 //	err = "output codec not found";
@@ -178,18 +149,18 @@ got_fn:
 //	av_init_packet(&output->avpkt);
 
 	// output frame and fifo
-	output->frame = av_frame_alloc();
-	output->frame->nb_samples = output->encoder->avcctx->frame_size ? : 256;
-	output->frame->format = output->encoder->avcctx->sample_fmt;
-	output->frame->sample_rate = output->encoder->avcctx->sample_rate;
-	output->frame->channel_layout = output->encoder->avcctx->channel_layout;
-	if (!output->frame->channel_layout)
-		output->frame->channel_layout = av_get_default_channel_layout(output->encoder->avcctx->channels);
-	if (av_frame_get_buffer(output->frame, 0) < 0)
-		abort();
-
-	output->fifo = av_audio_fifo_alloc(output->encoder->avcctx->sample_fmt, output->encoder->avcctx->channels,
-			output->frame->nb_samples);
+//	output->frame = av_frame_alloc();
+//	output->frame->nb_samples = output->encoder->avcctx->frame_size ? : 256;
+//	output->frame->format = output->encoder->avcctx->sample_fmt;
+//	output->frame->sample_rate = output->encoder->avcctx->sample_rate;
+//	output->frame->channel_layout = output->encoder->avcctx->channel_layout;
+//	if (!output->frame->channel_layout)
+//		output->frame->channel_layout = av_get_default_channel_layout(output->encoder->avcctx->channels);
+//	if (av_frame_get_buffer(output->frame, 0) < 0)
+//		abort();
+
+//	output->fifo = av_audio_fifo_alloc(output->encoder->avcctx->sample_fmt, output->encoder->avcctx->channels,
+//			output->frame->nb_samples);
 
 	db_config_stream(output);
 	return 0;
@@ -216,17 +187,17 @@ static void output_shutdown(output_t *output) {
 //	avcodec_free_context(&output->avcctx);
 //#endif
 	avformat_free_context(output->fmtctx);
-	av_audio_fifo_free(output->fifo);
-	av_frame_free(&output->frame);
+//	av_audio_fifo_free(output->fifo);
+//	av_frame_free(&output->frame);
 
 	encoder_close(output->encoder);
 
 //	output->avcctx = NULL;
 	output->fmtctx = NULL;
 	output->avst = NULL;
-	output->fifo = NULL;
+//	output->fifo = NULL;
 
-	output->fifo_pts = 0;
+//	output->fifo_pts = 0;
 
 //	format_init(&output->requested_format);
 //	format_init(&output->actual_format);

recording-daemon/types.h

@@ -117,10 +117,10 @@ struct output_s {
 	AVFormatContext *fmtctx;
 	AVStream *avst;
 //	AVPacket avpkt;
-	AVAudioFifo *fifo;
-	int64_t fifo_pts; // pts of first data in fifo
+//	AVAudioFifo *fifo;
+//	int64_t fifo_pts; // pts of first data in fifo
 //	int64_t mux_dts; // last dts passed to muxer
-	AVFrame *frame;
+//	AVFrame *frame;
 	encoder_t *encoder;
 };
 

utils/rtpengine-ng-client

@@ -48,6 +48,7 @@ GetOptions(
 	'codec-strip=s@'		=> \$options{'codec-strip'},
 	'codec-offer=s@'		=> \$options{'codec-offer'},
 	'codec-transcode=s@'		=> \$options{'codec-transcode'},
+	'ptime=i'			=> \$options{'ptime'},
 	'flags=s@'			=> \$options{'flags'},
 ) or die;
 
@@ -55,7 +56,7 @@ my $cmd = shift(@ARGV) or die;
 
 my %packet = (command => $cmd);
 
-for my $x (split(/,/, 'from-tag,to-tag,call-id,transport protocol,media address,ICE,address family,DTLS,via-branch,media address')) {
+for my $x (split(/,/, 'from-tag,to-tag,call-id,transport protocol,media address,ICE,address family,DTLS,via-branch,media address,ptime')) {
 	defined($options{$x}) and $packet{$x} = \$options{$x};
 }
 for my $x (split(/,/, 'TOS,delete-delay')) {