From 2845bb1efa30c93912b1c81f70f193d4ecd9eba6 Mon Sep 17 00:00:00 2001
From: Richard Fuchs <rfuchs@sipwise.com>
Date: Wed, 2 Jul 2025 11:50:56 -0400
Subject: [PATCH] MT#62571 streamline resampling

Move destination format information into the sink object.

Decode each codec directly to its matching output format and leave
resampling to the sinks.

Move managing of the adjusted multi-channel output format into the mix
context.

Make sure all inputs to a mix outputs use the same audio format.

Change-Id: Ib9f334443bfee26d59f2ede6e13ac80c66c1b57e
---
 recording-daemon/decoder.c  | 26 ++++++--------------------
 recording-daemon/decoder.h  |  5 +----
 recording-daemon/main.c     |  1 +
 recording-daemon/main.h     |  1 +
 recording-daemon/metafile.c |  4 ++--
 recording-daemon/mix.c      | 37 +++++++++++++++++++++++++++++++------
 recording-daemon/mix.h      |  1 +
 recording-daemon/output.c   | 22 ++++++++++++++++------
 recording-daemon/output.h   |  2 +-
 recording-daemon/packet.c   | 24 ++++++------------------
 recording-daemon/tls_send.c | 21 ++++++++-------------
 recording-daemon/types.h    |  2 +-
 12 files changed, 75 insertions(+), 71 deletions(-)

diff --git a/recording-daemon/decoder.c b/recording-daemon/decoder.c
index 2fa14ce89..84eccc162 100644
--- a/recording-daemon/decoder.c
+++ b/recording-daemon/decoder.c
@@ -22,12 +22,8 @@
 #include "tls_send.h"
 
 
-int resample_audio;
-
-
-
 // does not initialise the contained `sink`
-decode_t *decoder_new(const char *payload_str, const char *format, int ptime, const format_t *dec_format) {
+decode_t *decoder_new(const char *payload_str, const char *format, int ptime) {
 	char *slash = strchr(payload_str, '/');
 	if (!slash) {
 		ilog(LOG_WARN, "Invalid payload format: %s", payload_str);
@@ -65,22 +61,12 @@ decode_t *decoder_new(const char *payload_str, const char *format, int ptime, co
 	int rtp_clockrate = clockrate;
 	clockrate = fraction_mult(clockrate, &def->default_clockrate_fact);
 
-	// we can now config our output, which determines the sample format we convert to
 	format_t out_format = {
 		.clockrate = clockrate,
 		.channels = channels,
-		.format = -1,
+		.format = AV_SAMPLE_FMT_S16,
 	};
 
-	if (resample_audio)
-		out_format.clockrate = resample_audio;
-	// mono/stereo mixing goes here: out_format.channels = ...
-	// if the output has been configured already, re-use the same format
-	if (dec_format->format != -1)
-		out_format = *dec_format;
-	else
-		out_format.format = AV_SAMPLE_FMT_S16; // needed for TLS-only scenarios
-
 	str fmtp = STR(format);
 
 	decoder_t *dec = decoder_new_fmtp(def, rtp_clockrate, channels, ptime, &out_format, NULL, &fmtp, NULL);
@@ -106,21 +92,21 @@ static int decoder_got_frame(decoder_t *dec, AVFrame *frame, void *sp, void *dp)
 			(unsigned int) frame->extended_data[0][3]);
 
 	if (metafile->recording_on) {
-		sink_add(&deco->mix_sink, frame, &dec->dest_format);
+		sink_add(&deco->mix_sink, frame);
 
 		if (output) {
 			dbg("SSRC %lx of stream #%lu has single output", ssrc->ssrc, stream->id);
-			if (!sink_add(&output->sink, frame, &dec->dest_format))
+			if (!sink_add(&output->sink, frame))
 				ilog(LOG_ERR, "Failed to add decoded packet to individual output");
 		}
 	}
 
 	if (metafile->forwarding_on)
-		sink_add(&deco->tls_mix_sink, frame, &dec->dest_format);
+		sink_add(&deco->tls_mix_sink, frame);
 
 	if (ssrc->tls_fwd) {
 		dbg("SSRC %lx of stream #%lu has TLS forwarding stream", ssrc->ssrc, stream->id);
-		if (!sink_add(&ssrc->tls_fwd->sink, frame, &ssrc->tls_fwd->format))
+		if (!sink_add(&ssrc->tls_fwd->sink, frame))
 			ilog(LOG_ERR, "Failed to add decoded packet to TLS/TCP forward output");
 
 	}
diff --git a/recording-daemon/decoder.h b/recording-daemon/decoder.h
index 82f9b4a9d..5d3c7a9ea 100644
--- a/recording-daemon/decoder.h
+++ b/recording-daemon/decoder.h
@@ -5,10 +5,7 @@
 #include "str.h"
 
 
-extern int resample_audio;
-
-
-decode_t *decoder_new(const char *payload_str, const char *format, int ptime, const format_t *);
+decode_t *decoder_new(const char *payload_str, const char *format, int ptime);
 int decoder_input(decode_t *, const str *, unsigned long ts, ssrc_t *);
 void decoder_free(decode_t *);
 
diff --git a/recording-daemon/main.c b/recording-daemon/main.c
index 6f10f483b..aea0fca20 100644
--- a/recording-daemon/main.c
+++ b/recording-daemon/main.c
@@ -72,6 +72,7 @@ gboolean notify_record;
 gboolean notify_purge;
 gboolean mix_output_per_media = 0;
 gboolean flush_packets = 0;
+int resample_audio;
 
 static GQueue threads = G_QUEUE_INIT; // only accessed from main thread
 
diff --git a/recording-daemon/main.h b/recording-daemon/main.h
index a0c1874f1..232a9abef 100644
--- a/recording-daemon/main.h
+++ b/recording-daemon/main.h
@@ -56,6 +56,7 @@ extern gboolean notify_purge;
 extern gboolean mix_output_per_media;
 extern volatile int shutdown_flag;
 extern gboolean flush_packets;
+extern int resample_audio;
 
 extern struct rtpengine_common_config rtpe_common_config;
 
diff --git a/recording-daemon/metafile.c b/recording-daemon/metafile.c
index 6aef12a3a..b4536cd18 100644
--- a/recording-daemon/metafile.c
+++ b/recording-daemon/metafile.c
@@ -105,8 +105,6 @@ static void meta_mix_file_output(metafile_t *mf) {
 		return;
 
 	mf->mix_out = output_new_ext(mf, "mix", "mixed", "mix");
-	if (mix_method == MM_CHANNELS)
-		mf->mix_out->channel_mult = mix_num_inputs;
 	mf->mix = mix_new(&mf->mix_lock, &mf->mix_out->sink, mf->media_rec_slots);
 	db_do_stream(mf, mf->mix_out, NULL, 0);
 }
@@ -127,6 +125,8 @@ static void meta_mix_tls_output(metafile_t *mf) {
 
 	if (!tls_fwd_new(&mf->mix_tls_fwd))
 		return;
+	if (mix_method == MM_CHANNELS)
+		mf->mix_tls_fwd->sink.format.channels = mix_num_inputs;
 	if (!mf->tls_mix)
 		mf->tls_mix = mix_new(&mf->mix_lock, &mf->mix_tls_fwd->sink, mf->media_rec_slots);
 
diff --git a/recording-daemon/mix.c b/recording-daemon/mix.c
index 375fd7f68..8ff29af0d 100644
--- a/recording-daemon/mix.c
+++ b/recording-daemon/mix.c
@@ -239,8 +239,6 @@ static int mix_config_(mix_t *mix, const format_t *format) {
 		goto err;
 
 	mix->out_format = mix->in_format;
-	if (mix_method == MM_CHANNELS)
-		mix->out_format.channels *= mix_num_inputs;
 
 	return 0;
 
@@ -394,7 +392,7 @@ static int mix_add_(mix_t *mix, AVFrame *frame, unsigned int idx, void *ptr) {
 			else
 				goto err;
 		}
-		bool ok = sink_add(mix->sink, mix->sink_frame, &mix->out_format);
+		bool ok = sink_add(mix->sink, mix->sink_frame);
 
 		av_frame_unref(mix->sink_frame);
 
@@ -451,10 +449,24 @@ bool mix_config(sink_t *sink, const format_t *requested_format, format_t *actual
 	if (G_UNLIKELY(sink->mixer_idx == -1u))
 		sink->mixer_idx = mix_get_index(mix, ssrc, stream->media_sdp_id, stream->channel_slot);
 
-	if (!mix->sink->config(mix->sink, requested_format, actual_format))
-		return false;
+	if (mix->in_format.format == -1) {
+		format_t req_fmt = *requested_format;
+		if (mix_method == MM_CHANNELS)
+			req_fmt.channels *= mix_num_inputs;
+
+		if (!mix->sink->config(mix->sink, &req_fmt, actual_format))
+			return false;
 
-	mix_config_(mix, actual_format);
+		if (mix_method == MM_CHANNELS && actual_format->channels % mix_num_inputs == 0)
+			actual_format->channels /= mix_num_inputs;
+
+		mix_config_(mix, actual_format);
+	}
+	else {
+		// output to the format that's already set
+		sink->format = mix->in_format;
+		*actual_format = mix->in_format;
+	}
 
 	return true;
 }
@@ -469,3 +481,16 @@ void mix_close(mix_t *mix) {
 	mix->sink = NULL;
 	mix->lock = NULL;
 }
+
+
+void mix_sink_init(sink_t *sink, ssrc_t *ssrc, mix_t **mixp, int resample) {
+	sink_init(sink);
+	sink->ssrc = ssrc;
+	sink->mix = mixp;
+	sink->add = mix_add;
+	sink->config = mix_config;
+	if (mix_method == MM_CHANNELS)
+		sink->format.channels = 1;
+	if (resample > 0)
+		sink->format.clockrate = resample;
+}
diff --git a/recording-daemon/mix.h b/recording-daemon/mix.h
index 621e2f5d7..18ae785b4 100644
--- a/recording-daemon/mix.h
+++ b/recording-daemon/mix.h
@@ -13,5 +13,6 @@ void mix_set_channel_slots(mix_t *mix, unsigned int);
 bool mix_config(sink_t *, const format_t *requested_format, format_t *actual_format);
 bool mix_add(sink_t *, AVFrame *frame);
 unsigned int mix_get_index(mix_t *, void *, unsigned int, unsigned int);
+void mix_sink_init(sink_t *, ssrc_t *, mix_t **, int resample);
 
 #endif
diff --git a/recording-daemon/output.c b/recording-daemon/output.c
index e4f12d10b..fd3de25dc 100644
--- a/recording-daemon/output.c
+++ b/recording-daemon/output.c
@@ -13,6 +13,7 @@
 #include "recaux.h"
 #include "notify.h"
 #include "resample.h"
+#include "fix_frame_channel_layout.h"
 
 
 #define DEFAULT_AVIO_BUFSIZE 4096
@@ -45,12 +46,20 @@ static int output_got_packet(encoder_t *enc, void *u1, void *u2) {
 }
 
 
-bool sink_add(sink_t *sink, AVFrame *frame, const format_t *requested_format) {
+bool sink_add(sink_t *sink, AVFrame *frame) {
 	if (!sink)
 		return false;
 
+	// copy/init from frame
+	if (G_UNLIKELY(sink->format.format == -1))
+		sink->format.format = frame->format;
+	if (G_UNLIKELY(sink->format.channels == -1))
+		sink->format.channels = GET_CHANNELS(frame);
+	if (G_UNLIKELY(sink->format.clockrate == -1))
+		sink->format.clockrate = frame->sample_rate;
+
 	format_t actual_format;
-	if (!sink->config(sink, requested_format, &actual_format))
+	if (!sink->config(sink, &sink->format, &actual_format))
 		return false;
 
 	AVFrame *copy_frame = av_frame_clone(frame);
@@ -125,6 +134,7 @@ void sink_init(sink_t *sink) {
 	*sink = (__typeof(*sink)) {
 		.mixer_idx = -1u,
 	};
+	format_init(&sink->format);
 }
 
 static bool output_config__(sink_t *s, const format_t *requested_format, format_t *actual_format) {
@@ -138,7 +148,6 @@ static output_t *output_alloc(const char *path, const char *name) {
 	ret->full_filename = g_strdup_printf("%s/%s", path, name);
 	ret->file_format = output_file_format;
 	ret->encoder = encoder_new();
-	ret->channel_mult = 1;
 	ret->requested_format.format = -1;
 	ret->actual_format.format = -1;
 	ret->start_time_us = now_us();
@@ -301,6 +310,10 @@ output_t *output_new_ext(metafile_t *mf, const char *type, const char *kind, con
 	else
 		ret = output_new(output_path, mf, type, kind, label);
 
+	ret->sink.format.format = AV_SAMPLE_FMT_S16;
+	if (resample_audio > 0)
+		ret->sink.format.clockrate = resample_audio;
+
 	return ret;
 }
 
@@ -366,13 +379,10 @@ static bool output_config_(sink_t *sink, output_t *output, const format_t *reque
 
 	// mask the channel multiplier from external view
 	output->requested_format = *requested_format;
-	req_fmt.channels *= output->channel_mult;
 
 	if (encoder_config(output->encoder, output_codec, mp3_bitrate, 0, &req_fmt, &output->actual_format))
 		goto err;
 
-	if (output->actual_format.channels == req_fmt.channels)
-		output->actual_format.channels /= output->channel_mult;
 	// save the sample format
 	if (requested_format->format == -1)
 		output->requested_format.format = output->actual_format.format;
diff --git a/recording-daemon/output.h b/recording-daemon/output.h
index 44c376e4c..4a8460fe1 100644
--- a/recording-daemon/output.h
+++ b/recording-daemon/output.h
@@ -19,7 +19,7 @@ int output_config(output_t *output, const format_t *requested_format, format_t *
 void sink_init(sink_t *);
 void sink_close(sink_t *sink);
 
-bool sink_add(sink_t *, AVFrame *frame, const format_t *requested_format);
+bool sink_add(sink_t *, AVFrame *frame);
 
 
 #endif
diff --git a/recording-daemon/packet.c b/recording-daemon/packet.c
index e59331e7e..8ffcdf0f1 100644
--- a/recording-daemon/packet.c
+++ b/recording-daemon/packet.c
@@ -114,24 +114,12 @@ static void packet_decode(ssrc_t *ssrc, packet_t *packet) {
 		dbg("payload type for %u is %s", payload_type, payload_str);
 
 		pthread_mutex_lock(&mf->mix_lock);
-		format_t dec_format = { .format = -1 };
-		if (mf->mix_out)
-			dec_format = mf->mix_out->requested_format;
-		else if (ssrc->output)
-			dec_format = ssrc->output->requested_format;
-		ssrc->decoders[payload_type] = decoder_new(payload_str, format, ptime, &dec_format);
-
-		sink_init(&ssrc->decoders[payload_type]->mix_sink);
-		ssrc->decoders[payload_type]->mix_sink.ssrc = ssrc;
-		ssrc->decoders[payload_type]->mix_sink.mix = &mf->mix;
-		ssrc->decoders[payload_type]->mix_sink.add = mix_add;
-		ssrc->decoders[payload_type]->mix_sink.config = mix_config;
-
-		sink_init(&ssrc->decoders[payload_type]->tls_mix_sink);
-		ssrc->decoders[payload_type]->tls_mix_sink.ssrc = ssrc;
-		ssrc->decoders[payload_type]->tls_mix_sink.mix = &mf->tls_mix;
-		ssrc->decoders[payload_type]->tls_mix_sink.add = mix_add;
-		ssrc->decoders[payload_type]->tls_mix_sink.config = mix_config;
+		ssrc->decoders[payload_type] = decoder_new(payload_str, format, ptime);
+
+		mix_sink_init(&ssrc->decoders[payload_type]->mix_sink, ssrc, &mf->mix,
+				resample_audio);
+		mix_sink_init(&ssrc->decoders[payload_type]->tls_mix_sink, ssrc, &mf->tls_mix,
+				tls_resample);
 
 		pthread_mutex_unlock(&mf->mix_lock);
 		if (!ssrc->decoders[payload_type]) {
diff --git a/recording-daemon/tls_send.c b/recording-daemon/tls_send.c
index 456b7a906..958a518e6 100644
--- a/recording-daemon/tls_send.c
+++ b/recording-daemon/tls_send.c
@@ -237,13 +237,10 @@ static bool tls_fwd_add(sink_t *sink, AVFrame *frame) {
 }
 
 
-static bool tls_fwd_config(sink_t *sink, const format_t *requested_format, format_t *actual_format)
-{
-	*actual_format = (format_t) {
-		.clockrate = tls_resample,
-		.format = AV_SAMPLE_FMT_S16,
-		.channels = 1,
-	};
+static bool tls_fwd_config(sink_t *sink, const format_t *requested_format, format_t *actual_format) {
+	tls_fwd_t *tls_fwd = *sink->tls_fwd;
+	*actual_format = sink->format;
+	tls_fwd->format = sink->format;
 	return true;
 }
 
@@ -302,17 +299,15 @@ bool tls_fwd_new(tls_fwd_t **tlsp) {
 	if (!*tlsp) // may have been closed
 		return false;
 
-	tls_fwd->format = (format_t) {
-		.clockrate = tls_resample,
-		.channels = 1,
-		.format = AV_SAMPLE_FMT_S16,
-	};
-
 	sink_init(&tls_fwd->sink);
 	tls_fwd->sink.tls_fwd = tlsp;
 	tls_fwd->sink.add = tls_fwd_add;
 	tls_fwd->sink.config = tls_fwd_config;
 
+	tls_fwd->sink.format.format = AV_SAMPLE_FMT_S16;
+	tls_fwd->sink.format.clockrate = tls_resample;
+	tls_fwd->sink.format.channels = 1;
+
 	return true;
 }
 
diff --git a/recording-daemon/types.h b/recording-daemon/types.h
index 9bc69dd25..9c50dcf9c 100644
--- a/recording-daemon/types.h
+++ b/recording-daemon/types.h
@@ -60,6 +60,7 @@ struct sink_s {
 	};
 
 	resample_t resampler;
+	format_t format;
 
 	union {
 		unsigned int mixer_idx;
@@ -198,7 +199,6 @@ struct output_s {
 	const char *kind; // "mixed" or "single"
 	unsigned long long db_id;
 	gboolean skip_filename_extension;
-	unsigned int channel_mult;
 	int64_t start_time_us;
 
 	FILE *fp;