MT#62571 streamline resampling

Move destination format information into the sink object.

Decode each codec directly to its matching output format and leave
resampling to the sinks.

Move managing of the adjusted multi-channel output format into the mix
context.

Make sure all inputs to a mix outputs use the same audio format.

Change-Id: Ib9f334443bfee26d59f2ede6e13ac80c66c1b57e

recording-daemon/decoder.c

@@ -22,12 +22,8 @@
 #include "tls_send.h"
 
 
-int resample_audio;
-
-
-
 // does not initialise the contained `sink`
-decode_t *decoder_new(const char *payload_str, const char *format, int ptime, const format_t *dec_format) {
+decode_t *decoder_new(const char *payload_str, const char *format, int ptime) {
 	char *slash = strchr(payload_str, '/');
 	if (!slash) {
 		ilog(LOG_WARN, "Invalid payload format: %s", payload_str);
@@ -65,22 +61,12 @@ decode_t *decoder_new(const char *payload_str, const char *format, int ptime, co
 	int rtp_clockrate = clockrate;
 	clockrate = fraction_mult(clockrate, &def->default_clockrate_fact);
 
-	// we can now config our output, which determines the sample format we convert to
 	format_t out_format = {
 		.clockrate = clockrate,
 		.channels = channels,
-		.format = -1,
+		.format = AV_SAMPLE_FMT_S16,
 	};
 
-	if (resample_audio)
-		out_format.clockrate = resample_audio;
-	// mono/stereo mixing goes here: out_format.channels = ...
-	// if the output has been configured already, re-use the same format
-	if (dec_format->format != -1)
-		out_format = *dec_format;
-	else
-		out_format.format = AV_SAMPLE_FMT_S16; // needed for TLS-only scenarios
-
 	str fmtp = STR(format);
 
 	decoder_t *dec = decoder_new_fmtp(def, rtp_clockrate, channels, ptime, &out_format, NULL, &fmtp, NULL);
@@ -106,21 +92,21 @@ static int decoder_got_frame(decoder_t *dec, AVFrame *frame, void *sp, void *dp)
 			(unsigned int) frame->extended_data[0][3]);
 
 	if (metafile->recording_on) {
-		sink_add(&deco->mix_sink, frame, &dec->dest_format);
+		sink_add(&deco->mix_sink, frame);
 
 		if (output) {
 			dbg("SSRC %lx of stream #%lu has single output", ssrc->ssrc, stream->id);
-			if (!sink_add(&output->sink, frame, &dec->dest_format))
+			if (!sink_add(&output->sink, frame))
 				ilog(LOG_ERR, "Failed to add decoded packet to individual output");
 		}
 	}
 
 	if (metafile->forwarding_on)
-		sink_add(&deco->tls_mix_sink, frame, &dec->dest_format);
+		sink_add(&deco->tls_mix_sink, frame);
 
 	if (ssrc->tls_fwd) {
 		dbg("SSRC %lx of stream #%lu has TLS forwarding stream", ssrc->ssrc, stream->id);
-		if (!sink_add(&ssrc->tls_fwd->sink, frame, &ssrc->tls_fwd->format))
+		if (!sink_add(&ssrc->tls_fwd->sink, frame))
 			ilog(LOG_ERR, "Failed to add decoded packet to TLS/TCP forward output");
 
 	}

recording-daemon/decoder.h

@@ -5,10 +5,7 @@
 #include "str.h"
 
 
-extern int resample_audio;
-
-
-decode_t *decoder_new(const char *payload_str, const char *format, int ptime, const format_t *);
+decode_t *decoder_new(const char *payload_str, const char *format, int ptime);
 int decoder_input(decode_t *, const str *, unsigned long ts, ssrc_t *);
 void decoder_free(decode_t *);
 

recording-daemon/main.c

@@ -72,6 +72,7 @@ gboolean notify_record;
 gboolean notify_purge;
 gboolean mix_output_per_media = 0;
 gboolean flush_packets = 0;
+int resample_audio;
 
 static GQueue threads = G_QUEUE_INIT; // only accessed from main thread
 

recording-daemon/main.h

@@ -56,6 +56,7 @@ extern gboolean notify_purge;
 extern gboolean mix_output_per_media;
 extern volatile int shutdown_flag;
 extern gboolean flush_packets;
+extern int resample_audio;
 
 extern struct rtpengine_common_config rtpe_common_config;
 

recording-daemon/metafile.c

@@ -105,8 +105,6 @@ static void meta_mix_file_output(metafile_t *mf) {
 		return;
 
 	mf->mix_out = output_new_ext(mf, "mix", "mixed", "mix");
-	if (mix_method == MM_CHANNELS)
-		mf->mix_out->channel_mult = mix_num_inputs;
 	mf->mix = mix_new(&mf->mix_lock, &mf->mix_out->sink, mf->media_rec_slots);
 	db_do_stream(mf, mf->mix_out, NULL, 0);
 }
@@ -127,6 +125,8 @@ static void meta_mix_tls_output(metafile_t *mf) {
 
 	if (!tls_fwd_new(&mf->mix_tls_fwd))
 		return;
+	if (mix_method == MM_CHANNELS)
+		mf->mix_tls_fwd->sink.format.channels = mix_num_inputs;
 	if (!mf->tls_mix)
 		mf->tls_mix = mix_new(&mf->mix_lock, &mf->mix_tls_fwd->sink, mf->media_rec_slots);
 

recording-daemon/mix.c

@@ -239,8 +239,6 @@ static int mix_config_(mix_t *mix, const format_t *format) {
 		goto err;
 
 	mix->out_format = mix->in_format;
-	if (mix_method == MM_CHANNELS)
-		mix->out_format.channels *= mix_num_inputs;
 
 	return 0;
 
@@ -394,7 +392,7 @@ static int mix_add_(mix_t *mix, AVFrame *frame, unsigned int idx, void *ptr) {
 			else
 				goto err;
 		}
-		bool ok = sink_add(mix->sink, mix->sink_frame, &mix->out_format);
+		bool ok = sink_add(mix->sink, mix->sink_frame);
 
 		av_frame_unref(mix->sink_frame);
 
@@ -451,10 +449,24 @@ bool mix_config(sink_t *sink, const format_t *requested_format, format_t *actual
 	if (G_UNLIKELY(sink->mixer_idx == -1u))
 		sink->mixer_idx = mix_get_index(mix, ssrc, stream->media_sdp_id, stream->channel_slot);
 
-	if (!mix->sink->config(mix->sink, requested_format, actual_format))
-		return false;
+	if (mix->in_format.format == -1) {
+		format_t req_fmt = *requested_format;
+		if (mix_method == MM_CHANNELS)
+			req_fmt.channels *= mix_num_inputs;
+
+		if (!mix->sink->config(mix->sink, &req_fmt, actual_format))
+			return false;
 
-	mix_config_(mix, actual_format);
+		if (mix_method == MM_CHANNELS && actual_format->channels % mix_num_inputs == 0)
+			actual_format->channels /= mix_num_inputs;
+
+		mix_config_(mix, actual_format);
+	}
+	else {
+		// output to the format that's already set
+		sink->format = mix->in_format;
+		*actual_format = mix->in_format;
+	}
 
 	return true;
 }
@@ -469,3 +481,16 @@ void mix_close(mix_t *mix) {
 	mix->sink = NULL;
 	mix->lock = NULL;
 }
+
+
+void mix_sink_init(sink_t *sink, ssrc_t *ssrc, mix_t **mixp, int resample) {
+	sink_init(sink);
+	sink->ssrc = ssrc;
+	sink->mix = mixp;
+	sink->add = mix_add;
+	sink->config = mix_config;
+	if (mix_method == MM_CHANNELS)
+		sink->format.channels = 1;
+	if (resample > 0)
+		sink->format.clockrate = resample;
+}

recording-daemon/mix.h

@@ -13,5 +13,6 @@ void mix_set_channel_slots(mix_t *mix, unsigned int);
 bool mix_config(sink_t *, const format_t *requested_format, format_t *actual_format);
 bool mix_add(sink_t *, AVFrame *frame);
 unsigned int mix_get_index(mix_t *, void *, unsigned int, unsigned int);
+void mix_sink_init(sink_t *, ssrc_t *, mix_t **, int resample);
 
 #endif

recording-daemon/output.c

@@ -13,6 +13,7 @@
 #include "recaux.h"
 #include "notify.h"
 #include "resample.h"
+#include "fix_frame_channel_layout.h"
 
 
 #define DEFAULT_AVIO_BUFSIZE 4096
@@ -45,12 +46,20 @@ static int output_got_packet(encoder_t *enc, void *u1, void *u2) {
 }
 
 
-bool sink_add(sink_t *sink, AVFrame *frame, const format_t *requested_format) {
+bool sink_add(sink_t *sink, AVFrame *frame) {
 	if (!sink)
 		return false;
 
+	// copy/init from frame
+	if (G_UNLIKELY(sink->format.format == -1))
+		sink->format.format = frame->format;
+	if (G_UNLIKELY(sink->format.channels == -1))
+		sink->format.channels = GET_CHANNELS(frame);
+	if (G_UNLIKELY(sink->format.clockrate == -1))
+		sink->format.clockrate = frame->sample_rate;
+
 	format_t actual_format;
-	if (!sink->config(sink, requested_format, &actual_format))
+	if (!sink->config(sink, &sink->format, &actual_format))
 		return false;
 
 	AVFrame *copy_frame = av_frame_clone(frame);
@@ -125,6 +134,7 @@ void sink_init(sink_t *sink) {
 	*sink = (__typeof(*sink)) {
 		.mixer_idx = -1u,
 	};
+	format_init(&sink->format);
 }
 
 static bool output_config__(sink_t *s, const format_t *requested_format, format_t *actual_format) {
@@ -138,7 +148,6 @@ static output_t *output_alloc(const char *path, const char *name) {
 	ret->full_filename = g_strdup_printf("%s/%s", path, name);
 	ret->file_format = output_file_format;
 	ret->encoder = encoder_new();
-	ret->channel_mult = 1;
 	ret->requested_format.format = -1;
 	ret->actual_format.format = -1;
 	ret->start_time_us = now_us();
@@ -301,6 +310,10 @@ output_t *output_new_ext(metafile_t *mf, const char *type, const char *kind, con
 	else
 		ret = output_new(output_path, mf, type, kind, label);
 
+	ret->sink.format.format = AV_SAMPLE_FMT_S16;
+	if (resample_audio > 0)
+		ret->sink.format.clockrate = resample_audio;
+
 	return ret;
 }
 
@@ -366,13 +379,10 @@ static bool output_config_(sink_t *sink, output_t *output, const format_t *reque
 
 	// mask the channel multiplier from external view
 	output->requested_format = *requested_format;
-	req_fmt.channels *= output->channel_mult;
 
 	if (encoder_config(output->encoder, output_codec, mp3_bitrate, 0, &req_fmt, &output->actual_format))
 		goto err;
 
-	if (output->actual_format.channels == req_fmt.channels)
-		output->actual_format.channels /= output->channel_mult;
 	// save the sample format
 	if (requested_format->format == -1)
 		output->requested_format.format = output->actual_format.format;

recording-daemon/output.h

@@ -19,7 +19,7 @@ int output_config(output_t *output, const format_t *requested_format, format_t *
 void sink_init(sink_t *);
 void sink_close(sink_t *sink);
 
-bool sink_add(sink_t *, AVFrame *frame, const format_t *requested_format);
+bool sink_add(sink_t *, AVFrame *frame);
 
 
 #endif

recording-daemon/packet.c

@@ -114,24 +114,12 @@ static void packet_decode(ssrc_t *ssrc, packet_t *packet) {
 		dbg("payload type for %u is %s", payload_type, payload_str);
 
 		pthread_mutex_lock(&mf->mix_lock);
-		format_t dec_format = { .format = -1 };
-		if (mf->mix_out)
-			dec_format = mf->mix_out->requested_format;
-		else if (ssrc->output)
-			dec_format = ssrc->output->requested_format;
-		ssrc->decoders[payload_type] = decoder_new(payload_str, format, ptime, &dec_format);
-
-		sink_init(&ssrc->decoders[payload_type]->mix_sink);
-		ssrc->decoders[payload_type]->mix_sink.ssrc = ssrc;
-		ssrc->decoders[payload_type]->mix_sink.mix = &mf->mix;
-		ssrc->decoders[payload_type]->mix_sink.add = mix_add;
-		ssrc->decoders[payload_type]->mix_sink.config = mix_config;
-
-		sink_init(&ssrc->decoders[payload_type]->tls_mix_sink);
-		ssrc->decoders[payload_type]->tls_mix_sink.ssrc = ssrc;
-		ssrc->decoders[payload_type]->tls_mix_sink.mix = &mf->tls_mix;
-		ssrc->decoders[payload_type]->tls_mix_sink.add = mix_add;
-		ssrc->decoders[payload_type]->tls_mix_sink.config = mix_config;
+		ssrc->decoders[payload_type] = decoder_new(payload_str, format, ptime);
+
+		mix_sink_init(&ssrc->decoders[payload_type]->mix_sink, ssrc, &mf->mix,
+				resample_audio);
+		mix_sink_init(&ssrc->decoders[payload_type]->tls_mix_sink, ssrc, &mf->tls_mix,
+				tls_resample);
 
 		pthread_mutex_unlock(&mf->mix_lock);
 		if (!ssrc->decoders[payload_type]) {

recording-daemon/tls_send.c

@@ -237,13 +237,10 @@ static bool tls_fwd_add(sink_t *sink, AVFrame *frame) {
 }
 
 
-static bool tls_fwd_config(sink_t *sink, const format_t *requested_format, format_t *actual_format)
-{
-	*actual_format = (format_t) {
-		.clockrate = tls_resample,
-		.format = AV_SAMPLE_FMT_S16,
-		.channels = 1,
-	};
+static bool tls_fwd_config(sink_t *sink, const format_t *requested_format, format_t *actual_format) {
+	tls_fwd_t *tls_fwd = *sink->tls_fwd;
+	*actual_format = sink->format;
+	tls_fwd->format = sink->format;
 	return true;
 }
 
@@ -302,17 +299,15 @@ bool tls_fwd_new(tls_fwd_t **tlsp) {
 	if (!*tlsp) // may have been closed
 		return false;
 
-	tls_fwd->format = (format_t) {
-		.clockrate = tls_resample,
-		.channels = 1,
-		.format = AV_SAMPLE_FMT_S16,
-	};
-
 	sink_init(&tls_fwd->sink);
 	tls_fwd->sink.tls_fwd = tlsp;
 	tls_fwd->sink.add = tls_fwd_add;
 	tls_fwd->sink.config = tls_fwd_config;
 
+	tls_fwd->sink.format.format = AV_SAMPLE_FMT_S16;
+	tls_fwd->sink.format.clockrate = tls_resample;
+	tls_fwd->sink.format.channels = 1;
+
 	return true;
 }
 

recording-daemon/types.h

@@ -60,6 +60,7 @@ struct sink_s {
 	};
 
 	resample_t resampler;
+	format_t format;
 
 	union {
 		unsigned int mixer_idx;
@@ -198,7 +199,6 @@ struct output_s {
 	const char *kind; // "mixed" or "single"
 	unsigned long long db_id;
 	gboolean skip_filename_extension;
-	unsigned int channel_mult;
 	int64_t start_time_us;
 
 	FILE *fp;