From 3535d96fda897a42ae14db4221f57236d0233462 Mon Sep 17 00:00:00 2001 From: Richard Fuchs Date: Wed, 22 Dec 2021 14:29:01 -0500 Subject: [PATCH] TT#136957 fix TS tracking with supplemental codecs Supplemental codecs such as DTMF use static timestamps while the event is ongoing, leading to a TS jump when the RTP flow changes back to audio. The sequencer needs to be aware of this so it doesn't mistakenly see the next audio packet as overdue and starts to process it prematurely. Change-Id: I2faea9aceec21fc04920f6c3c94141725383379f --- daemon/codec.c | 29 ++++++++++++++++++++++++----- t/auto-daemon-tests.pl | 4 ++-- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/daemon/codec.c b/daemon/codec.c index a6556352e..451f66228 100644 --- a/daemon/codec.c +++ b/daemon/codec.c @@ -1458,14 +1458,20 @@ static int __handler_func_sequencer(struct media_packet *mp, struct transcode_pa packet = packet_sequencer_next_packet(&ssrc_in_p->sequencer); if (G_UNLIKELY(!packet)) { - if (!ch || !ch->encoder_format.clockrate || !ch->handler - || !ch->handler->dest_pt.codec_def) + if (!ch || !h->dest_pt.clock_rate || !ch->handler + || !h->dest_pt.codec_def) break; uint32_t ts_diff = packet_ts - ch->last_ts; + + // if packet TS is larger than last tracked TS, we can force the next packet if packets were lost and the TS + // difference is too large. if packet TS is the same or lower (can happen for supplement codecs) we can wait + // for the next packet + if (ts_diff == 0 || ts_diff >= 0x80000000) + break; + unsigned long long ts_diff_us = - (unsigned long long) ts_diff * 1000000 / ch->encoder_format.clockrate - * ch->handler->dest_pt.codec_def->clockrate_mult; + (unsigned long long) ts_diff * 1000000 / h->dest_pt.clock_rate; if (ts_diff_us >= 60000) { // arbitrary value packet = packet_sequencer_force_next_packet(&ssrc_in_p->sequencer); if (!packet) @@ -1477,6 +1483,20 @@ static int __handler_func_sequencer(struct media_packet *mp, struct transcode_pa break; } + uint32_t ts_diff = ch->last_ts - packet->ts; + if (ts_diff < 0x80000000) { // ch->last_ts >= packet->ts + // multiple consecutive packets with same TS: this could be a compound packet, e.g. a large video frame, or + // it could be a supplemental audio codec with static timestamps, in which case we adjust the TS forward + // by one frame length. This is needed so that the next real audio packet (with real TS) is not mistakenly + // seen as overdue + if (h->source_pt.codec_def && h->source_pt.codec_def->supplemental) + ch->last_ts += h->source_pt.clock_rate * (ch->ptime ?: 20) / 1000; + } + else + ch->last_ts = packet->ts; + + input_ch->last_ts = ch->last_ts; + // new packet might have different handlers h = packet->handler; if (ch) @@ -2904,7 +2924,6 @@ static int packet_decode(struct codec_ssrc_handler *ch, struct codec_ssrc_handle if (!ch->first_ts) ch->first_ts = packet->ts; - ch->last_ts = packet->ts; if (input_ch->dtmf_start_ts && !rtpe_config.dtmf_no_suppress) { if ((packet->ts > input_ch->dtmf_start_ts && packet->ts - input_ch->dtmf_start_ts > 80000) || diff --git a/t/auto-daemon-tests.pl b/t/auto-daemon-tests.pl index 96f3f796a..4d3cb08a9 100755 --- a/t/auto-daemon-tests.pl +++ b/t/auto-daemon-tests.pl @@ -13208,11 +13208,11 @@ rcv($sock_a, $port_b, rtpm(0, $seq+3, 4000+480, $ssrc, "\x40\xe0\xb3\xad\xbd\x3f snd($sock_b, $port_a, rtp(96, 2004, 4000+320, 0x5678, "\x08\x10\x01\xe0")); rcv($sock_a, $port_b, rtpm(0, $seq+4, 4000+640, $ssrc, "\xbd\xd3\x77\xd9\xc5\xd0\x44\x30\x32\x65\xb2\xab\xb8\x4c\x32\x35\x50\xcf\xd2\x70\x7a\xc6\xb8\xbe\x4c\x2e\x2d\x45\xb9\xac\xb4\xfd\x3c\x3d\x55\xf2\x5a\x47\x56\xc1\xb0\xb4\x71\x30\x2b\x3a\xc7\xb0\xb6\xd7\x4d\x50\xf6\x78\x45\x38\x41\xc7\xae\xae\xcc\x37\x2c\x36\xe5\xbb\xbd\xd7\x6d\xdb\xc9\xdd\x3f\x30\x36\xdc\xae\xab\xbd\x41\x2f\x37\x5d\xcb\xcf\x7b\xef\xc4\xb9\xc6\x42\x2d\x2e\x55\xb4\xac\xb8\x58\x39\x3d\x59\xea\x5c\x4a\x66\xbd\xb0\xb8\x50\x2e\x2c\x40\xbd\xaf\xb8\xe8\x48\x4e\x7d\x6b\x43\x3a\x4a\xbf\xad\xaf\xe4\x32\x2c\x3a\xcf\xb8\xbd\xdc\x66\xde\xcc\xf5\x3c\x30\x3b\xca\xad\xac\xc6\x3b\x2e\x39\x7c\xc6\xcd\xfa\xe7\xc3\xbb\xce\x3c\x2d\x31\xf2")); # test out of seq -snd($sock_b, $port_a, rtp(0, 2006, 4000+160*5, 0x5678, "\x00" x 160)); # processed because TS difference too large +snd($sock_b, $port_a, rtp(0, 2006, 4000+160*25, 0x5678, "\x00" x 160)); # processed because TS difference too large rcv($sock_a, $port_b, rtpm(0, $seq+6, 4000+160*5, $ssrc, "\x00" x 160)); snd($sock_b, $port_a, rtp(96, 2005, 4000+320, 0x5678, "\x08\x10\x01\xe0")); # repeat, no-op, dup, consumed # resume normal -snd($sock_b, $port_a, rtp(0, 2007, 4000+160*6, 0x5678, "\x00" x 160)); +snd($sock_b, $port_a, rtp(0, 2007, 4000+160*26, 0x5678, "\x00" x 160)); rcv($sock_a, $port_b, rtpm(0, $seq+7, 4000+160*6, $ssrc, "\x00" x 160)); # test TS reset snd($sock_b, $port_a, rtp(0, 2008, 2000, 0x5678, "\x00" x 160));