TT#5566 align mixed audio's timecodes to each other

Change-Id: Ib9f82ba7f56da80a0171b108bccdb56e5334ce94

recording-daemon/Makefile

@@ -5,7 +5,6 @@ CFLAGS+=	-std=c99
 CFLAGS+=	-D_GNU_SOURCE -D_POSIX_SOURCE -D_POSIX_C_SOURCE
 CFLAGS+=	`pkg-config --cflags glib-2.0`
 CFLAGS+=	`pkg-config --cflags gthread-2.0`
-CFLAGS+=	`pcre-config --cflags`
 CFLAGS+=	`pkg-config --cflags libavcodec`
 CFLAGS+=	`pkg-config --cflags libavformat`
 CFLAGS+=	`pkg-config --cflags libavutil`
@@ -15,7 +14,6 @@ CFLAGS+=	`pkg-config --cflags libavfilter`
 LDFLAGS=	-lm
 LDFLAGS+=	`pkg-config --libs glib-2.0`
 LDFLAGS+=	`pkg-config --libs gthread-2.0`
-LDFLAGS+=	`pcre-config --libs`
 LDFLAGS+=	`pkg-config --libs libavcodec`
 LDFLAGS+=	`pkg-config --libs libavformat`
 LDFLAGS+=	`pkg-config --libs libavutil`

recording-daemon/decoder.c

@@ -254,6 +254,7 @@ static int decoder_got_frame(decoder_t *dec, output_t *output, metafile_t *metaf
 		output_config(metafile->mix_out, dec->out_clockrate, dec->channels);
 		mix_config(metafile->mix, dec->out_clockrate, dec->channels);
 		AVFrame *clone = av_frame_clone(dec_frame);
+		clone->pts = dec_frame->pts;
 		if (mix_add(metafile->mix, clone, dec->mixer_idx, metafile->mix_out))
 			ilog(LOG_ERR, "Failed to add decoded packet to mixed output");
 	}
@@ -273,8 +274,8 @@ int decoder_input(decoder_t *dec, const str *data, unsigned long ts, output_t *o
 	if (G_UNLIKELY(!dec))
 		return -1;
 
-	dbg("%p dec pts %lu rtp_ts %lu incoming ts %lu", dec, (unsigned long) dec->pts,
-			(unsigned long) dec->rtp_ts, (unsigned long) ts);
+	dbg("%p dec pts %llu rtp_ts %llu incoming ts %lu", dec, (unsigned long long) dec->pts,
+			(unsigned long long) dec->rtp_ts, (unsigned long) ts);
 
 	if (G_UNLIKELY(dec->rtp_ts == (unsigned long) -1L)) {
 		// initialize pts
@@ -353,6 +354,11 @@ int decoder_input(decoder_t *dec, const str *data, unsigned long ts, output_t *o
 #endif
 
 		if (got_frame) {
+#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 0, 0)
+			dec->frame->pts = dec->frame->pkt_pts;
+#endif
+			if (G_UNLIKELY(dec->frame->pts == AV_NOPTS_VALUE))
+				dec->frame->pts = dec->avpkt.pts;
 			if (decoder_got_frame(dec, output, metafile))
 				return -1;
 		}

recording-daemon/metafile.c

@@ -19,9 +19,6 @@
 static pthread_mutex_t metafiles_lock = PTHREAD_MUTEX_INITIALIZER;
 static GHashTable *metafiles;
 
-//static pcre_t stream_interface_re,
-	      //stream_details_re;
-
 
 static void meta_free(void *ptr) {
 	metafile_t *mf = ptr;
@@ -263,8 +260,6 @@ void metafile_delete(char *name) {
 
 void metafile_setup(void) {
 	metafiles = g_hash_table_new(g_str_hash, g_str_equal);
-	//pcre_build(&stream_interface_re, "^STREAM (\\d+) interface$");
-	//pcre_build(&stream_interface_re, "^STREAM (\\d+) details$");
 }
 
 

recording-daemon/mix.c

@@ -12,13 +12,18 @@
 #include "output.h"
 
 
+#define NUM_INPUTS 2
+
+
 struct mix_s {
 	// format params
 	int clockrate;
 	int channels;
 
 	AVFilterGraph *graph;
-	AVFilterContext *src_ctxs[2];
+	AVFilterContext *src_ctxs[NUM_INPUTS];
+	uint64_t pts_offs[NUM_INPUTS]; // initialized at first input seen
+	uint64_t in_pts[NUM_INPUTS]; // running counter of last seen adjusted pts
 	AVFilterContext *amix_ctx;
 	AVFilterContext *sink_ctx;
 	unsigned int next_idx;
@@ -27,6 +32,9 @@ struct mix_s {
 	AVAudioResampleContext *avresample;
 	AVFrame *swr_frame;
 	int swr_buffers;
+	uint64_t out_pts; // starting at zero
+
+	AVFrame *silence_frame;
 };
 
 
@@ -39,7 +47,7 @@ static void mix_shutdown(mix_t *mix) {
 		avfilter_free(mix->sink_ctx);
 	mix->sink_ctx = NULL;
 
-	for (int i = 0; i < G_N_ELEMENTS(mix->src_ctxs); i++) {
+	for (int i = 0; i < NUM_INPUTS; i++) {
 		if (mix->src_ctxs[i])
 			avfilter_free(mix->src_ctxs[i]);
 		mix->src_ctxs[i] = NULL;
@@ -54,6 +62,7 @@ void mix_destroy(mix_t *mix) {
 	mix_shutdown(mix);
 	av_frame_free(&mix->sink_frame);
 	av_frame_free(&mix->swr_frame);
+	av_frame_free(&mix->silence_frame);
 	g_slice_free1(sizeof(*mix), mix);
 }
 
@@ -95,7 +104,7 @@ format_mismatch:
 	if (!flt)
 		goto err;
 
-	snprintf(args, sizeof(args), "inputs=%lu", (unsigned long) G_N_ELEMENTS(mix->src_ctxs));
+	snprintf(args, sizeof(args), "inputs=%lu", (unsigned long) NUM_INPUTS);
 	err = "failed to create amix filter context";
 	if (avfilter_graph_create_filter(&mix->amix_ctx, flt, NULL, args, NULL, mix->graph))
 		goto err;
@@ -106,7 +115,7 @@ format_mismatch:
 	if (!flt)
 		goto err;
 
-	for (int i = 0; i < G_N_ELEMENTS(mix->src_ctxs); i++) {
+	for (int i = 0; i < NUM_INPUTS; i++) {
 		dbg("init input ctx %i", i);
 
 		snprintf(args, sizeof(args), "time_base=%d/%d:sample_rate=%d:sample_fmt=%s:"
@@ -158,6 +167,9 @@ mix_t *mix_new() {
 	mix->channels = -1;
 	mix->sink_frame = av_frame_alloc();
 
+	for (int i = 0; i < NUM_INPUTS; i++)
+		mix->pts_offs[i] = (uint64_t) -1LL;
+
 	return mix;
 }
 
@@ -234,24 +246,81 @@ err:
 }
 
 
+static void mix_silence_fill_idx_off(mix_t *mix, unsigned int idx, unsigned int offset) {
+	while (mix->in_pts[idx] + offset < mix->out_pts) {
+		if (G_UNLIKELY(!mix->silence_frame)) {
+			mix->silence_frame = av_frame_alloc();
+			mix->silence_frame->format = AV_SAMPLE_FMT_S16;
+			mix->silence_frame->channel_layout =
+				av_get_default_channel_layout(mix->channels);
+			mix->silence_frame->nb_samples = mix->clockrate / 100;
+			mix->silence_frame->sample_rate = mix->clockrate;
+			if (av_frame_get_buffer(mix->silence_frame, 0) < 0) {
+				ilog(LOG_ERR, "Failed to get silence frame buffers");
+				return;
+			}
+			memset(mix->silence_frame->extended_data[0], 0, mix->silence_frame->linesize[0]);
+		}
+
+		dbg("pushing silence frame into stream %i (%lli < %llu)", idx,
+				(long long unsigned) mix->in_pts[idx],
+				(long long unsigned) mix->out_pts);
+
+		mix->silence_frame->pts = mix->in_pts[idx];
+		mix->in_pts[idx] += mix->silence_frame->nb_samples;
+
+		if (av_buffersrc_write_frame(mix->src_ctxs[idx], mix->silence_frame))
+			ilog(LOG_WARN, "Failed to write silence frame to buffer");
+	}
+}
+
+
+static void mix_silence_fill(mix_t *mix) {
+	for (int i = 0; i < NUM_INPUTS; i++) {
+		// check the pts of each input and give them max 1 second of delay.
+		// if they fall behind too much, fill input with silence. otherwise
+		// output stalls and won't produce media
+		mix_silence_fill_idx_off(mix, i, mix->clockrate);
+	}
+}
+
+
 // frees the frame passed to it
 int mix_add(mix_t *mix, AVFrame *frame, unsigned int idx, output_t *output) {
 	const char *err;
 
 	err = "index out of range";
-	if (idx >= G_N_ELEMENTS(mix->src_ctxs))
+	if (idx >= NUM_INPUTS)
 		goto err;
 
 	err = "mixer not initialized";
 	if (!mix->src_ctxs[idx])
 		goto err;
 
+	// adjust for media started late
+	if (G_UNLIKELY(mix->pts_offs[idx] == (uint64_t) -1LL))
+		mix->pts_offs[idx] = mix->out_pts - frame->pts;
+	frame->pts += mix->pts_offs[idx];
+
+	// fill missing time
+	mix_silence_fill_idx_off(mix, idx, 0);
+
+	uint64_t frame_pts = frame->pts; // because *_add_frame unrefs the frame and invalidates pts
+
 	err = "failed to add frame to mixer";
 	if (av_buffersrc_add_frame(mix->src_ctxs[idx], frame))
 		goto err;
 
+	// update running counters
+	if (frame_pts > mix->out_pts)
+		mix->out_pts = frame_pts;
+	if (frame_pts > mix->in_pts[idx])
+		mix->in_pts[idx] = frame_pts;
+
 	av_frame_free(&frame);
 
+	mix_silence_fill(mix);
+
 	while (1) {
 		int ret = av_buffersink_get_frame(mix->sink_ctx, mix->sink_frame);
 		err = "failed to get frame from mixer";
@@ -263,7 +332,7 @@ int mix_add(mix_t *mix, AVFrame *frame, unsigned int idx, output_t *output) {
 		}
 		frame = mix_resample_frame(mix, mix->sink_frame);
 
-		ret = output_add(output, mix->sink_frame);
+		ret = output_add(output, frame);
 
 		av_frame_unref(mix->sink_frame);
 

recording-daemon/pcre.c

@@ -1,14 +0,0 @@
-#include "pcre.h"
-#include <pcre.h>
-#include "log.h"
-
-
-//void pcre_build(pcre_t *out, const char *pattern) {
-//	const char *errptr;
-//	int erroff;
-//
-//	out->re = pcre_compile(pattern, PCRE_DOLLAR_ENDONLY | PCRE_DOTALL, &errptr, &erroff, NULL);
-//	if (!out->re)
-//		die("Failed to compile PCRE '%s': %s (at %i)", pattern, errptr, erroff);
-//	out->extra = pcre_study(out->re, 0, &errptr);
-//}

recording-daemon/pcre.h

@@ -1,8 +0,0 @@
-#ifndef _PCRE_H_
-#define _PCRE_H_
-
-#include "types.h"
-
-//void pcre_build(pcre_t *out, const char *pattern);
-
-#endif

recording-daemon/stream.c

@@ -12,6 +12,16 @@
 #include "packet.h"
 
 
+#define MAXBUFLEN 65535
+#ifndef AV_INPUT_BUFFER_PADDING_SIZE
+#define AV_INPUT_BUFFER_PADDING_SIZE 0
+#endif
+#ifndef FF_INPUT_BUFFER_PADDING_SIZE
+#define FF_INPUT_BUFFER_PADDING_SIZE 0
+#endif
+#define ALLOCLEN (MAXBUFLEN + AV_INPUT_BUFFER_PADDING_SIZE + FF_INPUT_BUFFER_PADDING_SIZE)
+
+
 // stream is locked
 void stream_close(stream_t *stream) {
 	if (stream->fd == -1)
@@ -37,17 +47,8 @@ static void stream_handler(handler_t *handler) {
 	if (stream->fd == -1)
 		goto out;
 
-	static const int maxbuflen = 65535;
-	static const int alloclen = maxbuflen
-#ifdef AV_INPUT_BUFFER_PADDING_SIZE
-		+ AV_INPUT_BUFFER_PADDING_SIZE
-#endif
-#ifdef FF_INPUT_BUFFER_PADDING_SIZE
-		+ FF_INPUT_BUFFER_PADDING_SIZE
-#endif
-		;
-	buf = malloc(alloclen);
-	int ret = read(stream->fd, buf, maxbuflen);
+	buf = malloc(ALLOCLEN);
+	int ret = read(stream->fd, buf, MAXBUFLEN);
 	if (ret == 0) {
 		ilog(LOG_INFO, "EOF on stream %s", stream->name);
 		stream_close(stream);

recording-daemon/types.h

@@ -5,7 +5,6 @@
 #include <pthread.h>
 #include <sys/types.h>
 #include <glib.h>
-#include <pcre.h>
 #include "str.h"
 
 
@@ -94,11 +93,4 @@ struct metafile_s {
 };
 
 
-// struct pcre_s {
-// 	pcre *re;
-// 	pcre_extra *extra;
-// };
-// typedef struct pcre_s pcre_t;
-
-
 #endif