TT#5566 support mixing all audio channels into one output

Change-Id: I0ffd8ba39fbda2c27e8bf7e6c36b965897f22c0c

debian/control

@@ -4,8 +4,9 @@ Priority: extra
 Maintainer: Sipwise Development Team <support@sipwise.com>
 Build-Depends: debhelper (>= 5),
                iptables-dev (>= 1.4),
-               libavformat-dev,
                libavcodec-dev,
+               libavfilter-dev,
+               libavformat-dev,
                libavresample-dev,
                libavutil-dev,
                libcurl4-openssl-dev | libcurl4-gnutls-dev | libcurl3-openssl-dev | libcurl3-gnutls-dev,

recording-daemon/Makefile

@@ -10,6 +10,7 @@ CFLAGS+=	`pkg-config --cflags libavcodec`
 CFLAGS+=	`pkg-config --cflags libavformat`
 CFLAGS+=	`pkg-config --cflags libavutil`
 CFLAGS+=	`pkg-config --cflags libavresample`
+CFLAGS+=	`pkg-config --cflags libavfilter`
 
 LDFLAGS=	-lm
 LDFLAGS+=	`pkg-config --libs glib-2.0`
@@ -19,11 +20,12 @@ LDFLAGS+=	`pkg-config --libs libavcodec`
 LDFLAGS+=	`pkg-config --libs libavformat`
 LDFLAGS+=	`pkg-config --libs libavutil`
 LDFLAGS+=	`pkg-config --libs libavresample`
+LDFLAGS+=	`pkg-config --libs libavfilter`
 
 include ../lib/lib.Makefile
 
 SRCS=		epoll.c garbage.c inotify.c main.c metafile.c stream.c recaux.c packet.c \
-		decoder.c output.c
+		decoder.c output.c mix.c
 LIBSRCS=	loglib.c auxlib.c rtplib.c
 OBJS=		$(SRCS:.c=.o) $(LIBSRCS:.c=.o)
 

recording-daemon/decoder.c

@@ -11,6 +11,7 @@
 #include "log.h"
 #include "str.h"
 #include "output.h"
+#include "mix.h"
 
 
 struct decoder_s {
@@ -28,6 +29,8 @@ struct decoder_s {
 	AVFrame *frame;
 	unsigned long rtp_ts;
 	uint64_t pts;
+
+	unsigned int mixer_idx;
 };
 
 
@@ -155,6 +158,7 @@ decoder_t *decoder_new(const char *payload_str) {
 
 	ret->pts = (uint64_t) -1LL;
 	ret->rtp_ts = (unsigned long) -1L;
+	ret->mixer_idx = (unsigned int) -1;
 
 	return ret;
 
@@ -178,12 +182,18 @@ static AVFrame *decoder_resample_frame(decoder_t *dec) {
 		if (!dec->avresample)
 			goto err;
 
-		av_opt_set_int(dec->avresample, "in_channel_layout", av_get_default_channel_layout(dec->channels), 0);
-		av_opt_set_int(dec->avresample, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
-		av_opt_set_int(dec->avresample, "in_sample_rate", dec->in_clockrate, 0);
-		av_opt_set_int(dec->avresample, "out_channel_layout", av_get_default_channel_layout(dec->channels), 0);
-		av_opt_set_int(dec->avresample, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
-		av_opt_set_int(dec->avresample, "out_sample_rate", dec->out_clockrate, 0);
+		av_opt_set_int(dec->avresample, "in_channel_layout",
+				av_get_default_channel_layout(dec->channels), 0);
+		av_opt_set_int(dec->avresample, "in_sample_fmt",
+				AV_SAMPLE_FMT_S16, 0);
+		av_opt_set_int(dec->avresample, "in_sample_rate",
+				dec->in_clockrate, 0);
+		av_opt_set_int(dec->avresample, "out_channel_layout",
+				av_get_default_channel_layout(dec->channels), 0);
+		av_opt_set_int(dec->avresample, "out_sample_fmt",
+				AV_SAMPLE_FMT_S16, 0);
+		av_opt_set_int(dec->avresample, "out_sample_rate",
+				dec->out_clockrate, 0);
 		// av_opt_set_int(dec->avresample, "internal_sample_fmt", AV_SAMPLE_FMT_FLTP, 0); // ?
 
 		err = "failed to init resample context";
@@ -232,19 +242,32 @@ err:
 }
 
 
-static int decoder_got_frame(decoder_t *dec, output_t *output) {
+static int decoder_got_frame(decoder_t *dec, output_t *output, metafile_t *metafile) {
 	// do we need to resample?
 	AVFrame *dec_frame = decoder_resample_frame(dec);
 
+	// handle mix output
+	pthread_mutex_lock(&metafile->mix_lock);
+	if (metafile->mix_out) {
+		if (G_UNLIKELY(dec->mixer_idx == (unsigned int) -1))
+			dec->mixer_idx = mix_get_index(metafile->mix);
+		output_config(metafile->mix_out, dec->out_clockrate, dec->channels);
+		mix_config(metafile->mix, dec->out_clockrate, dec->channels);
+		AVFrame *clone = av_frame_clone(dec_frame);
+		if (mix_add(metafile->mix, clone, dec->mixer_idx, metafile->mix_out))
+			ilog(LOG_ERR, "Failed to add decoded packet to mixed output");
+	}
+	pthread_mutex_unlock(&metafile->mix_lock);
+
 	output_config(output, dec->out_clockrate, dec->channels);
 	if (output_add(output, dec_frame))
-		return -1;
+		ilog(LOG_ERR, "Failed to add decoded packet to individual output");
 
 	return 0;
 }
 
 
-int decoder_input(decoder_t *dec, const str *data, unsigned long ts, output_t *output) {
+int decoder_input(decoder_t *dec, const str *data, unsigned long ts, output_t *output, metafile_t *metafile) {
 	const char *err;
 
 	if (G_UNLIKELY(!dec))
@@ -330,7 +353,7 @@ int decoder_input(decoder_t *dec, const str *data, unsigned long ts, output_t *o
 #endif
 
 		if (got_frame) {
-			if (decoder_got_frame(dec, output))
+			if (decoder_got_frame(dec, output, metafile))
 				return -1;
 		}
 	} while (keep_going);

recording-daemon/decoder.h

@@ -9,7 +9,7 @@ extern int resample_audio;
 
 
 decoder_t *decoder_new(const char *payload_str);
-int decoder_input(decoder_t *, const str *, unsigned long ts, output_t *);
+int decoder_input(decoder_t *, const str *, unsigned long ts, output_t *, metafile_t *);
 void decoder_close(decoder_t *);
 
 

recording-daemon/main.c

@@ -8,6 +8,7 @@
 #include <signal.h>
 #include <libavformat/avformat.h>
 #include <libavcodec/avcodec.h>
+#include <libavfilter/avfilter.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include "log.h"
@@ -63,6 +64,7 @@ static void setup(void) {
 	log_init();
 	av_register_all();
 	avcodec_register_all();
+	avfilter_register_all();
 	avformat_network_init();
 	signals();
 	metafile_setup();

recording-daemon/metafile.c

@@ -12,6 +12,8 @@
 #include "main.h"
 #include "recaux.h"
 #include "packet.h"
+#include "output.h"
+#include "mix.h"
 
 
 static pthread_mutex_t metafiles_lock = PTHREAD_MUTEX_INITIALIZER;
@@ -25,6 +27,8 @@ static void meta_free(void *ptr) {
 	metafile_t *mf = ptr;
 
 	dbg("freeing metafile info for %s", mf->name);
+	output_close(mf->mix_out);
+	mix_destroy(mf->mix);
 	g_string_chunk_free(mf->gsc);
 	for (int i = 0; i < mf->streams->len; i++) {
 		stream_t *stream = g_ptr_array_index(mf->streams, i);
@@ -51,6 +55,15 @@ static void meta_destroy(metafile_t *mf) {
 
 // mf is locked
 static void meta_stream_interface(metafile_t *mf, unsigned long snum, char *content) {
+	pthread_mutex_lock(&mf->mix_lock);
+	if (!mf->mix) {
+		char buf[256];
+		snprintf(buf, sizeof(buf), "%s/%s-mix", output_dir, mf->parent);
+		mf->mix_out = output_new(buf);
+		mf->mix = mix_new();
+	}
+	pthread_mutex_unlock(&mf->mix_lock);
+
 	dbg("stream %lu interface %s", snum, content);
 	stream_open(mf, snum, content);
 }
@@ -111,6 +124,7 @@ static metafile_t *metafile_get(char *name) {
 	mf->name = g_string_chunk_insert(mf->gsc, name);
 	pthread_mutex_init(&mf->lock, NULL);
 	pthread_mutex_init(&mf->payloads_lock, NULL);
+	pthread_mutex_init(&mf->mix_lock, NULL);
 	mf->streams = g_ptr_array_new();
 	mf->ssrc_hash = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, ssrc_free);
 

recording-daemon/mix.c

@@ -0,0 +1,267 @@
+#include "mix.h"
+#include <glib.h>
+#include <libavfilter/avfilter.h>
+#include <libavfilter/buffersrc.h>
+#include <libavfilter/buffersink.h>
+#include <libavutil/channel_layout.h>
+#include <inttypes.h>
+#include <libavresample/avresample.h>
+#include <libavutil/opt.h>
+#include "types.h"
+#include "log.h"
+#include "output.h"
+
+
+struct mix_s {
+	// format params
+	int clockrate;
+	int channels;
+
+	AVFilterGraph *graph;
+	AVFilterContext *src_ctxs[2];
+	AVFilterContext *amix_ctx;
+	AVFilterContext *sink_ctx;
+	unsigned int next_idx;
+
+	AVAudioResampleContext *avresample;
+	AVFrame *swr_frame;
+	int swr_buffers;
+};
+
+
+static void mix_shutdown(mix_t *mix) {
+	if (mix->amix_ctx)
+		avfilter_free(mix->amix_ctx);
+	mix->amix_ctx = NULL;
+
+	if (mix->sink_ctx)
+		avfilter_free(mix->sink_ctx);
+	mix->sink_ctx = NULL;
+
+	for (int i = 0; i < G_N_ELEMENTS(mix->src_ctxs); i++) {
+		if (mix->src_ctxs[i])
+			avfilter_free(mix->src_ctxs[i]);
+		mix->src_ctxs[i] = NULL;
+	}
+
+	avfilter_graph_free(&mix->graph);
+}
+
+
+void mix_destroy(mix_t *mix) {
+	mix_shutdown(mix);
+	g_slice_free1(sizeof(*mix), mix);
+}
+
+
+unsigned int mix_get_index(mix_t *mix) {
+	return mix->next_idx++;
+}
+
+
+int mix_config(mix_t *mix, unsigned int clockrate, unsigned int channels) {
+	const char *err;
+	char args[512];
+
+	// anything to do?
+	if (G_UNLIKELY(mix->clockrate != clockrate))
+		goto format_mismatch;
+	if (G_UNLIKELY(mix->channels != channels))
+		goto format_mismatch;
+
+	// all good
+	return 0;
+
+format_mismatch:
+	mix_shutdown(mix);
+
+	// copy params
+	mix->clockrate = clockrate;
+	mix->channels = channels;
+
+	// filter graph
+	err = "failed to alloc filter graph";
+	mix->graph = avfilter_graph_alloc();
+	if (!mix->graph)
+		goto err;
+
+	// amix
+	err = "no amix filter available";
+	AVFilter *flt = avfilter_get_by_name("amix");
+	if (!flt)
+		goto err;
+
+	snprintf(args, sizeof(args), "inputs=%lu", (unsigned long) G_N_ELEMENTS(mix->src_ctxs));
+	err = "failed to create amix filter context";
+	if (avfilter_graph_create_filter(&mix->amix_ctx, flt, NULL, args, NULL, mix->graph))
+		goto err;
+
+	// inputs
+	err = "no abuffer filter available";
+	flt = avfilter_get_by_name("abuffer");
+	if (!flt)
+		goto err;
+
+	for (int i = 0; i < G_N_ELEMENTS(mix->src_ctxs); i++) {
+		dbg("init input ctx %i", i);
+
+		snprintf(args, sizeof(args), "time_base=%d/%d:sample_rate=%d:sample_fmt=%s:"
+				"channel_layout=0x%" PRIx64,
+				1, mix->clockrate, mix->clockrate,
+				av_get_sample_fmt_name(AV_SAMPLE_FMT_S16),
+				av_get_default_channel_layout(mix->channels));
+
+		err = "failed to create abuffer filter context";
+		if (avfilter_graph_create_filter(&mix->src_ctxs[i], flt, NULL, args, NULL, mix->graph))
+			goto err;
+
+		err = "failed to link abuffer to amix";
+		if (avfilter_link(mix->src_ctxs[i], 0, mix->amix_ctx, i))
+			goto err;
+	}
+
+	// sink
+	err = "no abuffersink filter available";
+	flt = avfilter_get_by_name("abuffersink");
+	if (!flt)
+		goto err;
+
+	err = "failed to create abuffersink filter context";
+	if (avfilter_graph_create_filter(&mix->sink_ctx, flt, NULL, NULL, NULL, mix->graph))
+		goto err;
+
+	err = "failed to link amix to abuffersink";
+	if (avfilter_link(mix->amix_ctx, 0, mix->sink_ctx, 0))
+		goto err;
+
+	// finish up
+	err = "failed to configure filter chain";
+	if (avfilter_graph_config(mix->graph, NULL))
+		goto err;
+
+	return 0;
+
+err:
+	mix_shutdown(mix);
+	ilog(LOG_ERR, "Failed to initialize mixer: %s", err);
+	return -1;
+}
+
+
+mix_t *mix_new() {
+	mix_t *mix = g_slice_alloc0(sizeof(*mix));
+	mix->clockrate = -1;
+	mix->channels = -1;
+
+	return mix;
+}
+
+
+static AVFrame *mix_resample_frame(mix_t *mix, AVFrame *frame) {
+	const char *err;
+
+	if (frame->format == AV_SAMPLE_FMT_S16)
+		return frame;
+
+	if (!mix->avresample) {
+		mix->avresample = avresample_alloc_context();
+		err = "failed to alloc resample context";
+		if (!mix->avresample)
+			goto err;
+
+		av_opt_set_int(mix->avresample, "in_channel_layout",
+				av_get_default_channel_layout(mix->channels), 0);
+		av_opt_set_int(mix->avresample, "in_sample_fmt",
+				frame->format, 0);
+		av_opt_set_int(mix->avresample, "in_sample_rate",
+				mix->clockrate, 0);
+		av_opt_set_int(mix->avresample, "out_channel_layout",
+				av_get_default_channel_layout(mix->channels), 0);
+		av_opt_set_int(mix->avresample, "out_sample_fmt",
+				AV_SAMPLE_FMT_S16, 0);
+		av_opt_set_int(mix->avresample, "out_sample_rate",
+				mix->clockrate, 0);
+		// av_opt_set_int(dec->avresample, "internal_sample_fmt", AV_SAMPLE_FMT_FLTP, 0); // ?
+
+		err = "failed to init resample context";
+		if (avresample_open(mix->avresample) < 0)
+			goto err;
+	}
+
+	// get a large enough buffer for resampled audio - this should be enough so we don't
+	// have to loop
+	int dst_samples = avresample_available(mix->avresample) +
+		av_rescale_rnd(avresample_get_delay(mix->avresample) + frame->nb_samples,
+				mix->clockrate, mix->clockrate, AV_ROUND_UP);
+	if (!mix->swr_frame || mix->swr_buffers < dst_samples) {
+		av_frame_free(&mix->swr_frame);
+		mix->swr_frame = av_frame_alloc();
+		err = "failed to alloc resampling frame";
+		if (!mix->swr_frame)
+			goto err;
+		av_frame_copy_props(mix->swr_frame, frame);
+		mix->swr_frame->format = frame->format;
+		mix->swr_frame->channel_layout = frame->channel_layout;
+		mix->swr_frame->nb_samples = dst_samples;
+		mix->swr_frame->sample_rate = mix->clockrate;
+		err = "failed to get resample buffers";
+		if (av_frame_get_buffer(mix->swr_frame, 0) < 0)
+			goto err;
+		mix->swr_buffers = dst_samples;
+	}
+
+	mix->swr_frame->nb_samples = dst_samples;
+	int ret_samples = avresample_convert(mix->avresample, mix->swr_frame->extended_data,
+				mix->swr_frame->linesize[0], dst_samples,
+				frame->extended_data,
+				frame->linesize[0], frame->nb_samples);
+	err = "failed to resample audio";
+	if (ret_samples < 0)
+		goto err;
+
+	mix->swr_frame->nb_samples = ret_samples;
+	mix->swr_frame->pts = av_rescale(frame->pts, mix->clockrate, mix->clockrate);
+	return mix->swr_frame;
+
+err:
+	ilog(LOG_ERR, "Error resampling: %s", err);
+	return NULL;
+}
+
+
+int mix_add(mix_t *mix, AVFrame *frame, unsigned int idx, output_t *output) {
+	const char *err;
+
+	err = "index out of range";
+	if (idx >= G_N_ELEMENTS(mix->src_ctxs))
+		goto err;
+
+	err = "mixer not initialized";
+	if (!mix->src_ctxs[idx])
+		goto err;
+
+	err = "failed to add frame to mixer";
+	if (av_buffersrc_add_frame(mix->src_ctxs[idx], frame))
+		goto err;
+
+	while (1) {
+		int ret = av_buffersink_get_frame(mix->sink_ctx, frame);
+		err = "failed to get frame from mixer";
+		if (ret < 0) {
+			if (ret == AVERROR(EAGAIN))
+				break;
+			else
+				goto err;
+		}
+		frame = mix_resample_frame(mix, frame);
+
+		if (output_add(output, frame))
+			return -1;
+	}
+
+	return 0;
+
+err:
+	ilog(LOG_ERR, "Failed to add frame to mixer: %s", err);
+	return -1;
+}

recording-daemon/mix.h

@@ -0,0 +1,17 @@
+#ifndef _MIX_H_
+#define _MIX_H_
+
+#include "types.h"
+#include <libavutil/frame.h>
+
+
+mix_t *mix_new(void);
+void mix_destroy(mix_t *mix);
+
+int mix_config(mix_t *, unsigned int clockrate, unsigned int channels);
+int mix_add(mix_t *mix, AVFrame *frame, unsigned int idx, output_t *output);
+unsigned int mix_get_index(mix_t *);
+
+
+#endif
+

recording-daemon/output.c

@@ -203,7 +203,7 @@ format_mismatch:
 	output->avcctx->channel_layout = av_get_default_channel_layout(output->channels);
 	output->avcctx->sample_rate = output->clockrate;
 	output->avcctx->sample_fmt = AV_SAMPLE_FMT_S16;
-	output->avcctx->time_base = (AVRational){output->clockrate,1};
+	output->avcctx->time_base = (AVRational){1,output->clockrate};
 	output->avcctx->bit_rate = mp3_bitrate;
 	output->avst->time_base = output->avcctx->time_base;
 

recording-daemon/packet.c

@@ -164,7 +164,7 @@ static void packet_decode(ssrc_t *ssrc, packet_t *packet) {
 	}
 
 	if (decoder_input(ssrc->decoders[payload_type], &packet->payload, ntohl(packet->rtp->timestamp),
-			ssrc->output))
+			ssrc->output, ssrc->metafile))
 		ilog(LOG_ERR, "Failed to decode media packet");
 }
 

recording-daemon/types.h

@@ -23,6 +23,9 @@ struct decoder_s;
 typedef struct decoder_s decoder_t;
 struct output_s;
 typedef struct output_s output_t;
+struct mix_s;
+typedef struct mix_s mix_t;
+
 
 typedef void handler_func(handler_t *);
 
@@ -82,6 +85,10 @@ struct metafile_s {
 	GPtrArray *streams;
 	GHashTable *ssrc_hash; // contains ssrc_t objects
 
+	pthread_mutex_t mix_lock;
+	mix_t *mix;
+	output_t *mix_out;
+
 	pthread_mutex_t payloads_lock;
 	char *payload_types[128];
 };