From 7986ca0860455f061ff428942528bb224d69cb8d Mon Sep 17 00:00:00 2001 From: Richard Fuchs Date: Mon, 5 Feb 2018 15:26:25 -0500 Subject: [PATCH] TT#30900 support specifying codec parameters for transcoding Change-Id: Ifac093cfba74a7cfdf1ba22209d608e04fed8c10 --- README.md | 19 ++++++++++ daemon/codec.c | 95 +++++++++++++++++++++++++++++++++++--------------- daemon/sdp.c | 26 +++++++------- lib/codeclib.c | 74 +++++++++++++++++++++++++++++++++++++-- lib/codeclib.h | 9 +++++ lib/rtplib.h | 1 + lib/str.h | 6 ++-- 7 files changed, 180 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 59e5cdc3a..467bf5c36 100644 --- a/README.md +++ b/README.md @@ -1230,6 +1230,15 @@ Optionally included keys are: an `a=rtpmap` attribute, or can be from the list of RFC-defined codecs. Examples are `PCMU`, `opus`, or `telephone-event`. + It is possible to specify codec format parameters alongside with the codec name + in the same format as they're written in SDP for codecs that support them, + for example `opus/48000` to + specify Opus with 48 kHz sampling rate and one channel (mono), or + `opus/48000/2` for stereo Opus. If any format parameters are specified, + the codec will only be stripped if all of the format parameters match, and other + instances of the same codec with different format parameters will be left + untouched. + As a special keyword, `all` can be used to remove all codecs, except the ones that should explicitly offered (see below). Note that it is an error to strip all codecs and leave none that could be offered. In this case, the original @@ -1243,6 +1252,8 @@ Optionally included keys are: Codecs that were not present in the original list of codecs offered by the client will be ignored. + This list also supports codec format parameters as per above. + * `transcode` Similar to `offer` but allows codecs to be added to the list of offered codecs @@ -1258,6 +1269,14 @@ Optionally included keys are: list of offered codecs, then no transcoding will be done. Also note that if transcoding takes place, in-kernel forwarding is disabled for this media stream and all processing happens in userspace. + + If no codec format parameters are specified in this list (e.g. just `opus` + instead of `opus/48000/2`), default values will be chosen for them. + + For codecs that support different bitrates, it can be specified by appending + another slash followed by the bitrate in bits per second, + e.g. `opus/48000/2/32000`. In this case, all format parameters (clock rate, + channels) must also be specified. * `ptime` Contains an integer. If set, changes the `a=ptime` attribute's value in the outgoing diff --git a/daemon/codec.c b/daemon/codec.c index 37574cf9b..b09d49ff8 100644 --- a/daemon/codec.c +++ b/daemon/codec.c @@ -89,8 +89,8 @@ static void __make_transcoder(struct codec_handler *handler, struct rtp_payload_ goto reset; ilog(LOG_DEBUG, "Leaving transcode context for " STR_FORMAT "/%u/%i -> " STR_FORMAT "/%u/%i intact", - STR_FMT(&source->encoding), source->clock_rate, source->channels, - STR_FMT(&dest->encoding), dest->clock_rate, dest->channels); + STR_FMT(&source->encoding_with_params), source->clock_rate, source->channels, + STR_FMT(&dest->encoding_with_params), dest->clock_rate, dest->channels); return; @@ -105,8 +105,8 @@ reset: handler); ilog(LOG_DEBUG, "Created transcode context for " STR_FORMAT "/%u/%i -> " STR_FORMAT "/%u/%i", - STR_FMT(&source->encoding), source->clock_rate, source->channels, - STR_FMT(&dest->encoding), dest->clock_rate, dest->channels); + STR_FMT(&source->encoding_with_params), source->clock_rate, source->channels, + STR_FMT(&dest->encoding_with_params), dest->clock_rate, dest->channels); } static void __ensure_codec_def(struct rtp_payload_type *pt, struct call_media *media) { @@ -124,6 +124,7 @@ static GList *__delete_receiver_codec(struct call_media *receiver, GList *link) g_hash_table_remove(receiver->codecs_recv, &pt->payload_type); g_hash_table_remove(receiver->codec_names_recv, &pt->encoding); + g_hash_table_remove(receiver->codec_names_recv, &pt->encoding_with_params); GList *next = link->next; g_queue_delete_link(&receiver->codecs_prefs_recv, link); @@ -159,7 +160,7 @@ void codec_handlers_update(struct call_media *receiver, struct call_media *sink) pt->ptime = sink->ptime; if (!pref_dest_codec) { - ilog(LOG_DEBUG, "Default sink codec is " STR_FORMAT, STR_FMT(&pt->encoding)); + ilog(LOG_DEBUG, "Default sink codec is " STR_FORMAT, STR_FMT(&pt->encoding_with_params)); pref_dest_codec = pt; } } @@ -194,7 +195,7 @@ void codec_handlers_update(struct call_media *receiver, struct call_media *sink) if (pt->codec_def->avcodec_id != -1) { ilog(LOG_DEBUG, "Accepting offered codec " STR_FORMAT " due to transcoding", - STR_FMT(&pt->encoding)); + STR_FMT(&pt->encoding_with_params)); MEDIA_SET(receiver, TRANSCODE); } @@ -224,7 +225,7 @@ void codec_handlers_update(struct call_media *receiver, struct call_media *sink) // XXX sufficient to check against payload type? if (!g_hash_table_lookup(receiver->codec_names_send, &pt->encoding)) { ilog(LOG_DEBUG, "Eliminating transcoded codec " STR_FORMAT, - STR_FMT(&pt->encoding)); + STR_FMT(&pt->encoding_with_params)); l = __delete_receiver_codec(receiver, l); continue; @@ -235,7 +236,8 @@ void codec_handlers_update(struct call_media *receiver, struct call_media *sink) struct codec_handler *handler; handler = g_hash_table_lookup(receiver->codec_handlers, &pt->payload_type); if (!handler) { - ilog(LOG_DEBUG, "Creating codec handler for " STR_FORMAT, STR_FMT(&pt->encoding)); + ilog(LOG_DEBUG, "Creating codec handler for " STR_FORMAT, + STR_FMT(&pt->encoding_with_params)); handler = __handler_new(pt); g_hash_table_insert(receiver->codec_handlers, &handler->source_pt.payload_type, handler); @@ -261,7 +263,8 @@ void codec_handlers_update(struct call_media *receiver, struct call_media *sink) // to do. most likely this is an initial offer without a received answer. // we default to forwarding without transcoding. if (!pref_dest_codec) { - ilog(LOG_DEBUG, "No known/supported sink codec for " STR_FORMAT, STR_FMT(&pt->encoding)); + ilog(LOG_DEBUG, "No known/supported sink codec for " STR_FORMAT, + STR_FMT(&pt->encoding_with_params)); __make_passthrough(handler); goto next; } @@ -297,14 +300,14 @@ void codec_handlers_update(struct call_media *receiver, struct call_media *sink) } // XXX check format parameters as well - ilog(LOG_DEBUG, "Sink supports codec " STR_FORMAT, STR_FMT(&pt->encoding)); + ilog(LOG_DEBUG, "Sink supports codec " STR_FORMAT, STR_FMT(&pt->encoding_with_params)); __make_passthrough(handler); goto next; } unsupported: // the sink does not support this codec -> transcode - ilog(LOG_DEBUG, "Sink does not support codec " STR_FORMAT, STR_FMT(&pt->encoding)); + ilog(LOG_DEBUG, "Sink does not support codec " STR_FORMAT, STR_FMT(&pt->encoding_with_params)); dest_pt = pref_dest_codec; transcode: MEDIA_SET(receiver, TRANSCODE); @@ -410,9 +413,8 @@ static struct ssrc_entry *__ssrc_handler_new(void *p) { ch->encoder = encoder_new(); if (!ch->encoder) goto err; - // XXX make bitrate configurable if (encoder_config(ch->encoder, h->dest_pt.codec_def, - h->dest_pt.codec_def->default_bitrate, + h->dest_pt.bitrate ? : h->dest_pt.codec_def->default_bitrate, ch->ptime, &enc_format, &ch->encoder_format)) goto err; @@ -592,15 +594,22 @@ void codec_packet_free(void *pp) { -static struct rtp_payload_type *codec_make_dynamic_payload_type(const codec_def_t *dec, struct call *call) { +static struct rtp_payload_type *codec_make_dynamic_payload_type(const codec_def_t *dec, struct call_media *media, + int clockrate, int channels, int bitrate) +{ if (dec->default_channels <= 0 || dec->default_clockrate < 0) return NULL; struct rtp_payload_type *ret = g_slice_alloc0(sizeof(*ret)); ret->payload_type = -1; str_init(&ret->encoding, (char *) dec->rtpname); - ret->clock_rate = dec->default_clockrate; - ret->channels = dec->default_channels; + ret->clock_rate = clockrate ? : dec->default_clockrate; + ret->channels = channels ? : dec->default_channels; + ret->bitrate = bitrate; + ret->ptime = media->ptime ? : dec->default_ptime; + + if (dec->init) + dec->init(ret); char full_encoding[64]; char params[32] = ""; @@ -618,15 +627,29 @@ static struct rtp_payload_type *codec_make_dynamic_payload_type(const codec_def_ ret->format_parameters = STR_EMPTY; ret->codec_def = dec; - __rtp_payload_type_dup(call, ret); + __rtp_payload_type_dup(media->call, ret); + return ret; } -// XXX allow specifying codec params (e.g. "transcode=opus/16000/1") -static struct rtp_payload_type *codec_make_payload_type(const str *codec, struct call_media *media) { - struct call *call = media->call; - const codec_def_t *dec = codec_find(codec, media->type_id); +static struct rtp_payload_type *codec_make_payload_type(const str *codec_str, struct call_media *media) { + str codec_fmt = *codec_str; + str codec, parms, chans, opts; + if (str_token_sep(&codec, &codec_fmt, '/')) + return NULL; + str_token_sep(&parms, &codec_fmt, '/'); + str_token_sep(&chans, &codec_fmt, '/'); + str_token_sep(&opts, &codec_fmt, '/'); + + int clockrate = str_to_i(&parms, 0); + int channels = str_to_i(&chans, 0); + int bitrate = str_to_i(&opts, 0); + + if (clockrate && !channels) + channels = 1; + + const codec_def_t *dec = codec_find(&codec, media->type_id); if (!dec) return NULL; // we must support both encoding and decoding @@ -637,13 +660,17 @@ static struct rtp_payload_type *codec_make_payload_type(const str *codec, struct if (dec->rfc_payload_type >= 0) { const struct rtp_payload_type *rfc_pt = rtp_get_rfc_payload_type(dec->rfc_payload_type); - if (rfc_pt) { + // only use the RFC payload type if all parameters match + if (rfc_pt + && (clockrate == 0 || clockrate == rfc_pt->clock_rate) + && (channels == 0 || channels == rfc_pt->channels)) + { struct rtp_payload_type *ret = __rtp_payload_type_copy(rfc_pt); ret->codec_def = dec; return ret; } } - return codec_make_dynamic_payload_type(dec, call); + return codec_make_dynamic_payload_type(dec, media, clockrate, channels, bitrate); } @@ -667,7 +694,7 @@ static struct rtp_payload_type *codec_add_payload_type(const str *codec, struct else if (pt->payload_type >= 128) { ilog(LOG_WARN, "Ran out of RTP payload type numbers while adding codec '" STR_FORMAT "' for transcoding", - STR_FMT(codec)); + STR_FMT(&pt->encoding_with_params)); payload_type_free(pt); return NULL; } @@ -699,6 +726,8 @@ static void __rtp_payload_type_add_name(GHashTable *ht, struct rtp_payload_type { GQueue *q = g_hash_table_lookup_queue_new(ht, &pt->encoding); g_queue_push_tail(q, GUINT_TO_POINTER(pt->payload_type)); + q = g_hash_table_lookup_queue_new(ht, &pt->encoding_with_params); + g_queue_push_tail(q, GUINT_TO_POINTER(pt->payload_type)); } // consumes 'pt' static void __rtp_payload_type_add_recv(struct call_media *media, @@ -709,7 +738,9 @@ static void __rtp_payload_type_add_recv(struct call_media *media, g_queue_push_tail(&media->codecs_prefs_recv, pt); } // duplicates 'pt' -static void __rtp_payload_type_add_send(struct call_media *other_media, struct rtp_payload_type *pt) { +static void __rtp_payload_type_add_send(struct call_media *other_media, + struct rtp_payload_type *pt) +{ pt = __rtp_payload_type_copy(pt); g_hash_table_insert(other_media->codecs_send, &pt->payload_type, pt); __rtp_payload_type_add_name(other_media->codec_names_send, pt); @@ -728,7 +759,8 @@ static void __payload_queue_free(void *qq) { g_queue_free_full(q, (GDestroyNotify) payload_type_free); } static int __revert_codec_strip(GHashTable *removed, const str *codec, - struct call_media *media, struct call_media *other_media) { + struct call_media *media, struct call_media *other_media) +{ GQueue *q = g_hash_table_lookup(removed, codec); if (!q) return 0; @@ -772,9 +804,14 @@ void codec_rtp_payload_types(struct call_media *media, struct call_media *other_ // codec stripping if (strip) { - if (remove_all || g_hash_table_lookup(strip, &pt->encoding)) { - ilog(LOG_DEBUG, "Stripping codec '" STR_FORMAT "'", STR_FMT(&pt->encoding)); + if (remove_all || g_hash_table_lookup(strip, &pt->encoding) + || g_hash_table_lookup(strip, &pt->encoding_with_params)) + { + ilog(LOG_DEBUG, "Stripping codec '" STR_FORMAT "'", + STR_FMT(&pt->encoding_with_params)); GQueue *q = g_hash_table_lookup_queue_new(removed, &pt->encoding); + g_queue_push_tail(q, __rtp_payload_type_copy(pt)); + q = g_hash_table_lookup_queue_new(removed, &pt->encoding_with_params); g_queue_push_tail(q, pt); continue; } @@ -809,7 +846,7 @@ void codec_rtp_payload_types(struct call_media *media, struct call_media *other_ continue; ilog(LOG_DEBUG, "Codec '" STR_FORMAT "' added for transcoding with payload type %u", - STR_FMT(codec), pt->payload_type); + STR_FMT(&pt->encoding_with_params), pt->payload_type); __rtp_payload_type_add_recv(media, pt); } diff --git a/daemon/sdp.c b/daemon/sdp.c index d40a9bcfc..29adc8c5b 100644 --- a/daemon/sdp.c +++ b/daemon/sdp.c @@ -1128,7 +1128,6 @@ static int __rtp_payload_types(struct stream_params *sp, struct sdp_media *media GList *ql; struct sdp_attribute *attr; int ret = 0; - int ptime = 0; if (!sp->protocol || !sp->protocol->rtp) return 0; @@ -1150,11 +1149,6 @@ static int __rtp_payload_types(struct stream_params *sp, struct sdp_media *media g_hash_table_insert(ht_fmtp, &attr->u.fmtp.payload_type, &attr->u.fmtp.format_parms_str); } - // check for a=ptime - attr = attr_get_by_id(&media->attributes, ATTR_PTIME); - if (attr && attr->value.s) - ptime = atoi(attr->value.s); - /* then go through the format list and associate */ for (ql = media->format_list.head; ql; ql = ql->next) { char *ep; @@ -1186,8 +1180,8 @@ static int __rtp_payload_types(struct stream_params *sp, struct sdp_media *media pt->format_parameters = *s; // fill in ptime - if (ptime) - pt->ptime = ptime; + if (sp->ptime) + pt->ptime = sp->ptime; else if (!pt->ptime && ptrfc) pt->ptime = ptrfc->ptime; @@ -1284,6 +1278,11 @@ int sdp_streams(const GQueue *sessions, GQueue *streams, struct sdp_ng_flags *fl bf_set_clear(&sp->sp_flags, SP_FLAG_STRICT_SOURCE, flags->strict_source); bf_set_clear(&sp->sp_flags, SP_FLAG_MEDIA_HANDOVER, flags->media_handover); + // a=ptime + attr = attr_get_by_id(&media->attributes, ATTR_PTIME); + if (attr && attr->value.s) + sp->ptime = str_to_i(&attr->value, 0); + errstr = "Invalid RTP payload types"; if (__rtp_payload_types(sp, media)) goto error; @@ -1343,11 +1342,6 @@ int sdp_streams(const GQueue *sessions, GQueue *streams, struct sdp_ng_flags *fl __sdp_ice(sp, media); - // a=ptime - attr = attr_get_by_id(&media->attributes, ATTR_PTIME); - if (attr && attr->value.s) - sp->ptime = atoi(attr->value.s); - /* determine RTCP endpoint */ if (attr_get_by_id(&media->attributes, ATTR_RTCP_MUX)) { @@ -1715,10 +1709,14 @@ static int process_media_attributes(struct sdp_chopper *chop, struct sdp_media * case ATTR_SENDRECV: case ATTR_IGNORE: case ATTR_END_OF_CANDIDATES: // we strip it here and re-insert it later + goto strip; + case ATTR_RTPMAP: case ATTR_FMTP: case ATTR_PTIME: - goto strip; + if (media->codecs_prefs_recv.length > 0) + goto strip; + break; case ATTR_EXTMAP: case ATTR_CRYPTO: diff --git a/lib/codeclib.c b/lib/codeclib.c index 90dff11a8..15cfd348f 100644 --- a/lib/codeclib.c +++ b/lib/codeclib.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include "str.h" #include "log.h" @@ -27,8 +28,11 @@ -packetizer_f packetizer_passthrough; // pass frames as they arrive in AVPackets -packetizer_f packetizer_samplestream; // flat stream of samples +static packetizer_f packetizer_passthrough; // pass frames as they arrive in AVPackets +static packetizer_f packetizer_samplestream; // flat stream of samples + +static format_init_f opus_init; +static set_options_f opus_set_options; @@ -131,6 +135,8 @@ static codec_def_t __codec_defs[] = { .default_ptime = 20, .packetizer = packetizer_passthrough, .type = MT_AUDIO, + .init = opus_init, + .set_options = opus_set_options, }, { .rtpname = "vorbis", @@ -736,6 +742,7 @@ int encoder_config(encoder_t *enc, const codec_def_t *def, int bitrate, int ptim enc->requested_format = *requested_format; err = "output codec not found"; + enc->def = def; enc->codec = def->encoder; // if (codec_name) // enc->codec = avcodec_find_encoder_by_name(codec_name); @@ -745,6 +752,7 @@ int encoder_config(encoder_t *enc, const codec_def_t *def, int bitrate, int ptim goto err; ptime /= def->clockrate_mult; + enc->ptime = ptime; err = "failed to alloc codec context"; enc->avcctx = avcodec_alloc_context3(enc->codec); @@ -769,6 +777,10 @@ int encoder_config(encoder_t *enc, const codec_def_t *def, int bitrate, int ptim enc->avcctx->sample_fmt = enc->actual_format.format; enc->avcctx->time_base = (AVRational){1,enc->actual_format.clockrate}; enc->avcctx->bit_rate = bitrate; + enc->samples_per_frame = enc->actual_format.clockrate * ptime / 1000; + + if (def->set_options) + def->set_options(enc); err = "failed to open output context"; int i = avcodec_open2(enc->avcctx, enc->codec, NULL); @@ -777,7 +789,6 @@ int encoder_config(encoder_t *enc, const codec_def_t *def, int bitrate, int ptim av_init_packet(&enc->avpkt); - enc->samples_per_frame = enc->actual_format.clockrate * ptime / 1000; // output frame and fifo enc->frame = av_frame_alloc(); @@ -979,3 +990,60 @@ int packetizer_samplestream(AVPacket *pkt, GString *buf, str *input_output) { g_string_erase(buf, 0, input_output->len); return buf->len >= input_output->len ? 1 : 0; } + + +static void opus_init(struct rtp_payload_type *pt) { + if (pt->clock_rate != 48000) { + ilog(LOG_WARN, "Opus is only supported with a clock rate of 48 kHz"); + pt->clock_rate = 48000; + } + + switch (pt->ptime) { + case 5: + case 10: + case 20: + case 40: + case 60: + break; + default: + ; + int np; + if (pt->ptime < 10) + np = 5; + else if (pt->ptime < 20) + np = 10; + else if (pt->ptime < 40) + np = 20; + else if (pt->ptime < 60) + np = 40; + else + np = 60; + ilog(LOG_INFO, "Opus doesn't support a ptime of %i ms; using %i ms instead", + pt->ptime, np); + pt->ptime = np; + break; + } + + if (pt->bitrate) { + if (pt->bitrate < 6000) { + ilog(LOG_DEBUG, "Opus bitrate %i bps too small, assuming %i kbit/s", + pt->bitrate, pt->bitrate); + pt->bitrate *= 1000; + } + return; + } + if (pt->channels == 1) + pt->bitrate = 24000; + else if (pt->channels == 2) + pt->bitrate = 32000; + else + pt->bitrate = 64000; + ilog(LOG_DEBUG, "Using default bitrate of %i bps for %i-channel Opus", pt->bitrate, pt->channels); +} + +static void opus_set_options(encoder_t *enc) { + int ret; + if ((ret = av_opt_set_int(enc->avcctx, "frame_duration", enc->ptime, 0))) + ilog(LOG_WARN, "Failed to set Opus frame_duration option (error code %i)", ret); + // XXX additional opus options +} diff --git a/lib/codeclib.h b/lib/codeclib.h index 9bbc78f8f..372940575 100644 --- a/lib/codeclib.h +++ b/lib/codeclib.h @@ -16,6 +16,7 @@ struct format_s; struct resample_s; struct seq_packet_s; struct packet_sequencer_s; +struct rtp_payload_type; typedef struct codec_def_s codec_def_t; typedef struct decoder_s decoder_t; @@ -26,6 +27,8 @@ typedef struct seq_packet_s seq_packet_t; typedef struct packet_sequencer_s packet_sequencer_t; typedef int packetizer_f(AVPacket *, GString *, str *); +typedef void format_init_f(struct rtp_payload_type *); +typedef void set_options_f(encoder_t *); @@ -51,6 +54,10 @@ struct codec_def_s { const int decode_only_ok; const enum media_type type; + // codec-specific callbacks + format_init_f *init; + set_options_f *set_options; + // filled in by codeclib_init() str rtpname_str; int rfc_payload_type; @@ -87,11 +94,13 @@ struct encoder_s { format_t requested_format, actual_format; + const codec_def_t *def; AVCodec *codec; AVCodecContext *avcctx; AVPacket avpkt; AVAudioFifo *fifo; int64_t fifo_pts; // pts of first data in fifo + int ptime; int samples_per_frame; AVFrame *frame; // to pull samples from the fifo int64_t mux_dts; // last dts passed to muxer diff --git a/lib/rtplib.h b/lib/rtplib.h index 4d13a5266..2b6e0cee7 100644 --- a/lib/rtplib.h +++ b/lib/rtplib.h @@ -26,6 +26,7 @@ struct rtp_payload_type { str format_parameters; // value of a=fmtp int ptime; // default from RFC + int bitrate; const codec_def_t *codec_def; }; diff --git a/lib/str.h b/lib/str.h index 0ebe2cf4b..73b722040 100644 --- a/lib/str.h +++ b/lib/str.h @@ -280,8 +280,6 @@ INLINE void str_swap(str *a, str *b) { INLINE int str_to_i(str *s, int def) { char c, *ep; long ret; - int maxint = 0x7FFFFFFF; - int minint = 0x80000000; if (s->len <= 0) return def; c = s->s[s->len]; @@ -290,9 +288,9 @@ INLINE int str_to_i(str *s, int def) { s->s[s->len] = c; if (ep == s->s) return def; - if (ret > maxint) + if (ret > INT_MAX) return def; - if (ret < minint) + if (ret < INT_MIN) return def; return ret; }