diff --git a/debian/control b/debian/control index a6b516cda..a9b0b7aa2 100644 --- a/debian/control +++ b/debian/control @@ -4,8 +4,9 @@ Priority: extra Maintainer: Sipwise Development Team Build-Depends: debhelper (>= 5), iptables-dev (>= 1.4), - libavformat-dev, libavcodec-dev, + libavfilter-dev, + libavformat-dev, libavresample-dev, libavutil-dev, libcurl4-openssl-dev | libcurl4-gnutls-dev | libcurl3-openssl-dev | libcurl3-gnutls-dev, diff --git a/recording-daemon/Makefile b/recording-daemon/Makefile index d8084154d..49348537a 100644 --- a/recording-daemon/Makefile +++ b/recording-daemon/Makefile @@ -10,6 +10,7 @@ CFLAGS+= `pkg-config --cflags libavcodec` CFLAGS+= `pkg-config --cflags libavformat` CFLAGS+= `pkg-config --cflags libavutil` CFLAGS+= `pkg-config --cflags libavresample` +CFLAGS+= `pkg-config --cflags libavfilter` LDFLAGS= -lm LDFLAGS+= `pkg-config --libs glib-2.0` @@ -19,11 +20,12 @@ LDFLAGS+= `pkg-config --libs libavcodec` LDFLAGS+= `pkg-config --libs libavformat` LDFLAGS+= `pkg-config --libs libavutil` LDFLAGS+= `pkg-config --libs libavresample` +LDFLAGS+= `pkg-config --libs libavfilter` include ../lib/lib.Makefile SRCS= epoll.c garbage.c inotify.c main.c metafile.c stream.c recaux.c packet.c \ - decoder.c output.c + decoder.c output.c mix.c LIBSRCS= loglib.c auxlib.c rtplib.c OBJS= $(SRCS:.c=.o) $(LIBSRCS:.c=.o) diff --git a/recording-daemon/decoder.c b/recording-daemon/decoder.c index 640918452..373b608b8 100644 --- a/recording-daemon/decoder.c +++ b/recording-daemon/decoder.c @@ -11,6 +11,7 @@ #include "log.h" #include "str.h" #include "output.h" +#include "mix.h" struct decoder_s { @@ -28,6 +29,8 @@ struct decoder_s { AVFrame *frame; unsigned long rtp_ts; uint64_t pts; + + unsigned int mixer_idx; }; @@ -155,6 +158,7 @@ decoder_t *decoder_new(const char *payload_str) { ret->pts = (uint64_t) -1LL; ret->rtp_ts = (unsigned long) -1L; + ret->mixer_idx = (unsigned int) -1; return ret; @@ -178,12 +182,18 @@ static AVFrame *decoder_resample_frame(decoder_t *dec) { if (!dec->avresample) goto err; - av_opt_set_int(dec->avresample, "in_channel_layout", av_get_default_channel_layout(dec->channels), 0); - av_opt_set_int(dec->avresample, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0); - av_opt_set_int(dec->avresample, "in_sample_rate", dec->in_clockrate, 0); - av_opt_set_int(dec->avresample, "out_channel_layout", av_get_default_channel_layout(dec->channels), 0); - av_opt_set_int(dec->avresample, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0); - av_opt_set_int(dec->avresample, "out_sample_rate", dec->out_clockrate, 0); + av_opt_set_int(dec->avresample, "in_channel_layout", + av_get_default_channel_layout(dec->channels), 0); + av_opt_set_int(dec->avresample, "in_sample_fmt", + AV_SAMPLE_FMT_S16, 0); + av_opt_set_int(dec->avresample, "in_sample_rate", + dec->in_clockrate, 0); + av_opt_set_int(dec->avresample, "out_channel_layout", + av_get_default_channel_layout(dec->channels), 0); + av_opt_set_int(dec->avresample, "out_sample_fmt", + AV_SAMPLE_FMT_S16, 0); + av_opt_set_int(dec->avresample, "out_sample_rate", + dec->out_clockrate, 0); // av_opt_set_int(dec->avresample, "internal_sample_fmt", AV_SAMPLE_FMT_FLTP, 0); // ? err = "failed to init resample context"; @@ -232,19 +242,32 @@ err: } -static int decoder_got_frame(decoder_t *dec, output_t *output) { +static int decoder_got_frame(decoder_t *dec, output_t *output, metafile_t *metafile) { // do we need to resample? AVFrame *dec_frame = decoder_resample_frame(dec); + // handle mix output + pthread_mutex_lock(&metafile->mix_lock); + if (metafile->mix_out) { + if (G_UNLIKELY(dec->mixer_idx == (unsigned int) -1)) + dec->mixer_idx = mix_get_index(metafile->mix); + output_config(metafile->mix_out, dec->out_clockrate, dec->channels); + mix_config(metafile->mix, dec->out_clockrate, dec->channels); + AVFrame *clone = av_frame_clone(dec_frame); + if (mix_add(metafile->mix, clone, dec->mixer_idx, metafile->mix_out)) + ilog(LOG_ERR, "Failed to add decoded packet to mixed output"); + } + pthread_mutex_unlock(&metafile->mix_lock); + output_config(output, dec->out_clockrate, dec->channels); if (output_add(output, dec_frame)) - return -1; + ilog(LOG_ERR, "Failed to add decoded packet to individual output"); return 0; } -int decoder_input(decoder_t *dec, const str *data, unsigned long ts, output_t *output) { +int decoder_input(decoder_t *dec, const str *data, unsigned long ts, output_t *output, metafile_t *metafile) { const char *err; if (G_UNLIKELY(!dec)) @@ -330,7 +353,7 @@ int decoder_input(decoder_t *dec, const str *data, unsigned long ts, output_t *o #endif if (got_frame) { - if (decoder_got_frame(dec, output)) + if (decoder_got_frame(dec, output, metafile)) return -1; } } while (keep_going); diff --git a/recording-daemon/decoder.h b/recording-daemon/decoder.h index 239bab401..8eb659dcf 100644 --- a/recording-daemon/decoder.h +++ b/recording-daemon/decoder.h @@ -9,7 +9,7 @@ extern int resample_audio; decoder_t *decoder_new(const char *payload_str); -int decoder_input(decoder_t *, const str *, unsigned long ts, output_t *); +int decoder_input(decoder_t *, const str *, unsigned long ts, output_t *, metafile_t *); void decoder_close(decoder_t *); diff --git a/recording-daemon/main.c b/recording-daemon/main.c index 3ad7643ed..2350fb2c9 100644 --- a/recording-daemon/main.c +++ b/recording-daemon/main.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include "log.h" @@ -63,6 +64,7 @@ static void setup(void) { log_init(); av_register_all(); avcodec_register_all(); + avfilter_register_all(); avformat_network_init(); signals(); metafile_setup(); diff --git a/recording-daemon/metafile.c b/recording-daemon/metafile.c index a7556b9e8..453c6087d 100644 --- a/recording-daemon/metafile.c +++ b/recording-daemon/metafile.c @@ -12,6 +12,8 @@ #include "main.h" #include "recaux.h" #include "packet.h" +#include "output.h" +#include "mix.h" static pthread_mutex_t metafiles_lock = PTHREAD_MUTEX_INITIALIZER; @@ -25,6 +27,8 @@ static void meta_free(void *ptr) { metafile_t *mf = ptr; dbg("freeing metafile info for %s", mf->name); + output_close(mf->mix_out); + mix_destroy(mf->mix); g_string_chunk_free(mf->gsc); for (int i = 0; i < mf->streams->len; i++) { stream_t *stream = g_ptr_array_index(mf->streams, i); @@ -51,6 +55,15 @@ static void meta_destroy(metafile_t *mf) { // mf is locked static void meta_stream_interface(metafile_t *mf, unsigned long snum, char *content) { + pthread_mutex_lock(&mf->mix_lock); + if (!mf->mix) { + char buf[256]; + snprintf(buf, sizeof(buf), "%s/%s-mix", output_dir, mf->parent); + mf->mix_out = output_new(buf); + mf->mix = mix_new(); + } + pthread_mutex_unlock(&mf->mix_lock); + dbg("stream %lu interface %s", snum, content); stream_open(mf, snum, content); } @@ -111,6 +124,7 @@ static metafile_t *metafile_get(char *name) { mf->name = g_string_chunk_insert(mf->gsc, name); pthread_mutex_init(&mf->lock, NULL); pthread_mutex_init(&mf->payloads_lock, NULL); + pthread_mutex_init(&mf->mix_lock, NULL); mf->streams = g_ptr_array_new(); mf->ssrc_hash = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, ssrc_free); diff --git a/recording-daemon/mix.c b/recording-daemon/mix.c new file mode 100644 index 000000000..dca504ddb --- /dev/null +++ b/recording-daemon/mix.c @@ -0,0 +1,267 @@ +#include "mix.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "types.h" +#include "log.h" +#include "output.h" + + +struct mix_s { + // format params + int clockrate; + int channels; + + AVFilterGraph *graph; + AVFilterContext *src_ctxs[2]; + AVFilterContext *amix_ctx; + AVFilterContext *sink_ctx; + unsigned int next_idx; + + AVAudioResampleContext *avresample; + AVFrame *swr_frame; + int swr_buffers; +}; + + +static void mix_shutdown(mix_t *mix) { + if (mix->amix_ctx) + avfilter_free(mix->amix_ctx); + mix->amix_ctx = NULL; + + if (mix->sink_ctx) + avfilter_free(mix->sink_ctx); + mix->sink_ctx = NULL; + + for (int i = 0; i < G_N_ELEMENTS(mix->src_ctxs); i++) { + if (mix->src_ctxs[i]) + avfilter_free(mix->src_ctxs[i]); + mix->src_ctxs[i] = NULL; + } + + avfilter_graph_free(&mix->graph); +} + + +void mix_destroy(mix_t *mix) { + mix_shutdown(mix); + g_slice_free1(sizeof(*mix), mix); +} + + +unsigned int mix_get_index(mix_t *mix) { + return mix->next_idx++; +} + + +int mix_config(mix_t *mix, unsigned int clockrate, unsigned int channels) { + const char *err; + char args[512]; + + // anything to do? + if (G_UNLIKELY(mix->clockrate != clockrate)) + goto format_mismatch; + if (G_UNLIKELY(mix->channels != channels)) + goto format_mismatch; + + // all good + return 0; + +format_mismatch: + mix_shutdown(mix); + + // copy params + mix->clockrate = clockrate; + mix->channels = channels; + + // filter graph + err = "failed to alloc filter graph"; + mix->graph = avfilter_graph_alloc(); + if (!mix->graph) + goto err; + + // amix + err = "no amix filter available"; + AVFilter *flt = avfilter_get_by_name("amix"); + if (!flt) + goto err; + + snprintf(args, sizeof(args), "inputs=%lu", (unsigned long) G_N_ELEMENTS(mix->src_ctxs)); + err = "failed to create amix filter context"; + if (avfilter_graph_create_filter(&mix->amix_ctx, flt, NULL, args, NULL, mix->graph)) + goto err; + + // inputs + err = "no abuffer filter available"; + flt = avfilter_get_by_name("abuffer"); + if (!flt) + goto err; + + for (int i = 0; i < G_N_ELEMENTS(mix->src_ctxs); i++) { + dbg("init input ctx %i", i); + + snprintf(args, sizeof(args), "time_base=%d/%d:sample_rate=%d:sample_fmt=%s:" + "channel_layout=0x%" PRIx64, + 1, mix->clockrate, mix->clockrate, + av_get_sample_fmt_name(AV_SAMPLE_FMT_S16), + av_get_default_channel_layout(mix->channels)); + + err = "failed to create abuffer filter context"; + if (avfilter_graph_create_filter(&mix->src_ctxs[i], flt, NULL, args, NULL, mix->graph)) + goto err; + + err = "failed to link abuffer to amix"; + if (avfilter_link(mix->src_ctxs[i], 0, mix->amix_ctx, i)) + goto err; + } + + // sink + err = "no abuffersink filter available"; + flt = avfilter_get_by_name("abuffersink"); + if (!flt) + goto err; + + err = "failed to create abuffersink filter context"; + if (avfilter_graph_create_filter(&mix->sink_ctx, flt, NULL, NULL, NULL, mix->graph)) + goto err; + + err = "failed to link amix to abuffersink"; + if (avfilter_link(mix->amix_ctx, 0, mix->sink_ctx, 0)) + goto err; + + // finish up + err = "failed to configure filter chain"; + if (avfilter_graph_config(mix->graph, NULL)) + goto err; + + return 0; + +err: + mix_shutdown(mix); + ilog(LOG_ERR, "Failed to initialize mixer: %s", err); + return -1; +} + + +mix_t *mix_new() { + mix_t *mix = g_slice_alloc0(sizeof(*mix)); + mix->clockrate = -1; + mix->channels = -1; + + return mix; +} + + +static AVFrame *mix_resample_frame(mix_t *mix, AVFrame *frame) { + const char *err; + + if (frame->format == AV_SAMPLE_FMT_S16) + return frame; + + if (!mix->avresample) { + mix->avresample = avresample_alloc_context(); + err = "failed to alloc resample context"; + if (!mix->avresample) + goto err; + + av_opt_set_int(mix->avresample, "in_channel_layout", + av_get_default_channel_layout(mix->channels), 0); + av_opt_set_int(mix->avresample, "in_sample_fmt", + frame->format, 0); + av_opt_set_int(mix->avresample, "in_sample_rate", + mix->clockrate, 0); + av_opt_set_int(mix->avresample, "out_channel_layout", + av_get_default_channel_layout(mix->channels), 0); + av_opt_set_int(mix->avresample, "out_sample_fmt", + AV_SAMPLE_FMT_S16, 0); + av_opt_set_int(mix->avresample, "out_sample_rate", + mix->clockrate, 0); + // av_opt_set_int(dec->avresample, "internal_sample_fmt", AV_SAMPLE_FMT_FLTP, 0); // ? + + err = "failed to init resample context"; + if (avresample_open(mix->avresample) < 0) + goto err; + } + + // get a large enough buffer for resampled audio - this should be enough so we don't + // have to loop + int dst_samples = avresample_available(mix->avresample) + + av_rescale_rnd(avresample_get_delay(mix->avresample) + frame->nb_samples, + mix->clockrate, mix->clockrate, AV_ROUND_UP); + if (!mix->swr_frame || mix->swr_buffers < dst_samples) { + av_frame_free(&mix->swr_frame); + mix->swr_frame = av_frame_alloc(); + err = "failed to alloc resampling frame"; + if (!mix->swr_frame) + goto err; + av_frame_copy_props(mix->swr_frame, frame); + mix->swr_frame->format = frame->format; + mix->swr_frame->channel_layout = frame->channel_layout; + mix->swr_frame->nb_samples = dst_samples; + mix->swr_frame->sample_rate = mix->clockrate; + err = "failed to get resample buffers"; + if (av_frame_get_buffer(mix->swr_frame, 0) < 0) + goto err; + mix->swr_buffers = dst_samples; + } + + mix->swr_frame->nb_samples = dst_samples; + int ret_samples = avresample_convert(mix->avresample, mix->swr_frame->extended_data, + mix->swr_frame->linesize[0], dst_samples, + frame->extended_data, + frame->linesize[0], frame->nb_samples); + err = "failed to resample audio"; + if (ret_samples < 0) + goto err; + + mix->swr_frame->nb_samples = ret_samples; + mix->swr_frame->pts = av_rescale(frame->pts, mix->clockrate, mix->clockrate); + return mix->swr_frame; + +err: + ilog(LOG_ERR, "Error resampling: %s", err); + return NULL; +} + + +int mix_add(mix_t *mix, AVFrame *frame, unsigned int idx, output_t *output) { + const char *err; + + err = "index out of range"; + if (idx >= G_N_ELEMENTS(mix->src_ctxs)) + goto err; + + err = "mixer not initialized"; + if (!mix->src_ctxs[idx]) + goto err; + + err = "failed to add frame to mixer"; + if (av_buffersrc_add_frame(mix->src_ctxs[idx], frame)) + goto err; + + while (1) { + int ret = av_buffersink_get_frame(mix->sink_ctx, frame); + err = "failed to get frame from mixer"; + if (ret < 0) { + if (ret == AVERROR(EAGAIN)) + break; + else + goto err; + } + frame = mix_resample_frame(mix, frame); + + if (output_add(output, frame)) + return -1; + } + + return 0; + +err: + ilog(LOG_ERR, "Failed to add frame to mixer: %s", err); + return -1; +} diff --git a/recording-daemon/mix.h b/recording-daemon/mix.h new file mode 100644 index 000000000..e631cf73f --- /dev/null +++ b/recording-daemon/mix.h @@ -0,0 +1,17 @@ +#ifndef _MIX_H_ +#define _MIX_H_ + +#include "types.h" +#include + + +mix_t *mix_new(void); +void mix_destroy(mix_t *mix); + +int mix_config(mix_t *, unsigned int clockrate, unsigned int channels); +int mix_add(mix_t *mix, AVFrame *frame, unsigned int idx, output_t *output); +unsigned int mix_get_index(mix_t *); + + +#endif + diff --git a/recording-daemon/output.c b/recording-daemon/output.c index c4111c494..440010039 100644 --- a/recording-daemon/output.c +++ b/recording-daemon/output.c @@ -203,7 +203,7 @@ format_mismatch: output->avcctx->channel_layout = av_get_default_channel_layout(output->channels); output->avcctx->sample_rate = output->clockrate; output->avcctx->sample_fmt = AV_SAMPLE_FMT_S16; - output->avcctx->time_base = (AVRational){output->clockrate,1}; + output->avcctx->time_base = (AVRational){1,output->clockrate}; output->avcctx->bit_rate = mp3_bitrate; output->avst->time_base = output->avcctx->time_base; diff --git a/recording-daemon/packet.c b/recording-daemon/packet.c index af5f54fd4..b4dd11387 100644 --- a/recording-daemon/packet.c +++ b/recording-daemon/packet.c @@ -164,7 +164,7 @@ static void packet_decode(ssrc_t *ssrc, packet_t *packet) { } if (decoder_input(ssrc->decoders[payload_type], &packet->payload, ntohl(packet->rtp->timestamp), - ssrc->output)) + ssrc->output, ssrc->metafile)) ilog(LOG_ERR, "Failed to decode media packet"); } diff --git a/recording-daemon/types.h b/recording-daemon/types.h index 12694778e..19ead37d9 100644 --- a/recording-daemon/types.h +++ b/recording-daemon/types.h @@ -23,6 +23,9 @@ struct decoder_s; typedef struct decoder_s decoder_t; struct output_s; typedef struct output_s output_t; +struct mix_s; +typedef struct mix_s mix_t; + typedef void handler_func(handler_t *); @@ -82,6 +85,10 @@ struct metafile_s { GPtrArray *streams; GHashTable *ssrc_hash; // contains ssrc_t objects + pthread_mutex_t mix_lock; + mix_t *mix; + output_t *mix_out; + pthread_mutex_t payloads_lock; char *payload_types[128]; };