From 4b7ec4e11fb5a01abf8ba656ca21ddd904925489 Mon Sep 17 00:00:00 2001 From: Richard Fuchs Date: Mon, 27 Feb 2023 14:55:50 -0500 Subject: [PATCH] MT#56471 add flag to suppress early media Track audio writes in the mix buffer to set the `active` flag to true whenever a write occurs, which makes it possible to create the buffer in an inactivate state and implicitly set it active on demand. Handle the mix buffer not returning any data in the RTP sending logic (which is what happens for an inactive buffer) by simply not sending any packets. Change-Id: Iaeb0f6deadb3d90020c8c62872735cc94db80504 --- daemon/audio_player.c | 20 ++++++++++++++++++-- daemon/call.c | 3 +++ daemon/call_interfaces.c | 3 +++ daemon/codec.c | 2 ++ docs/ng_control_protocol.md | 8 ++++++++ include/audio_player.h | 2 ++ include/call_interfaces.h | 1 + lib/mix_buffer.c | 11 +++++++---- lib/mix_buffer.h | 11 +++++++++-- utils/rtpengine-ng-client | 1 + 10 files changed, 54 insertions(+), 8 deletions(-) diff --git a/daemon/audio_player.c b/daemon/audio_player.c index f3f79926e..ab5c829ba 100644 --- a/daemon/audio_player.c +++ b/daemon/audio_player.c @@ -33,6 +33,12 @@ static bool audio_player_run(struct media_player *mp) { unsigned int size; void *buf = mix_buffer_read_fast(&ap->mb, ap->ptime, &size); if (!buf) { + if (!size) { + // error or not active: just reschedule + timeval_add_usec(&mp->next_run, ap->ptime_us); + timerthread_obj_schedule_abs(&mp->tt_obj, &mp->next_run); + return false; + } buf = g_alloca(size); mix_buffer_read_slow(&ap->mb, buf, ap->ptime); } @@ -64,7 +70,6 @@ bool audio_player_setup(struct call_media *m, const struct rtp_payload_type *dst unsigned int ptime_smp = ptime_ms * clockrate / 1000; // in samples // TODO: shortcut this to avoid the detour of avframe -> avpacket -> avframe (all in s16) - // TODO: determine dest sample format from created encoder struct rtp_payload_type src_pt = { .payload_type = -1, .encoding = STR_CONST_INIT("PCM-S16LE"), // XXX support flp @@ -121,7 +126,8 @@ bool audio_player_setup(struct call_media *m, const struct rtp_payload_type *dst bufsize_ms = MAX(bufsize_ms, ptime_ms * 2); // make sure the buf size is at least 2 frames - mix_buffer_init(&ap->mb, AV_SAMPLE_FMT_S16, clockrate, dst_pt->channels, bufsize_ms, delay_ms); + mix_buffer_init_active(&ap->mb, AV_SAMPLE_FMT_S16, clockrate, dst_pt->channels, bufsize_ms, delay_ms, + false); return true; @@ -131,6 +137,16 @@ error: } +void audio_player_activate(struct call_media *m) { + if (!m) + return; + struct audio_player *ap = m->audio_player; + if (!ap) + return; + mix_buffer_activate(&ap->mb); +} + + // call locked in W void audio_player_start(struct call_media *m) { struct audio_player *ap; diff --git a/daemon/call.c b/daemon/call.c index 4919a9491..8e30eca1e 100644 --- a/daemon/call.c +++ b/daemon/call.c @@ -2759,6 +2759,9 @@ void codecs_offer_answer(struct call_media *media, struct call_media *other_medi set_transcoding_flag(media->monologue, other_media->monologue, true); if (codec_handlers_update(other_media, media, NULL, NULL)) set_transcoding_flag(other_media->monologue, media->monologue, true); + + // activate audio player if needed (not done by codec_handlers_update without `flags`) + audio_player_activate(media); } } diff --git a/daemon/call_interfaces.c b/daemon/call_interfaces.c index ce7d4ff11..232254ee5 100644 --- a/daemon/call_interfaces.c +++ b/daemon/call_interfaces.c @@ -975,6 +975,9 @@ static void call_ng_flags_flags(struct sdp_ng_flags *out, str *s, void *dummy) { case CSH_LOOKUP("reset"): out->reset = 1; break; + case CSH_LOOKUP("early-media"): + out->early_media = 1; + break; case CSH_LOOKUP("all"): out->all = ALL_ALL; break; diff --git a/daemon/codec.c b/daemon/codec.c index f040c4785..574ea267b 100644 --- a/daemon/codec.c +++ b/daemon/codec.c @@ -1438,6 +1438,8 @@ next: audio_player_setup(sink, pref_dest_codec, rtpe_config.audio_buffer_length, rtpe_config.audio_buffer_delay); + if (flags && (flags->early_media || flags->opmode == OP_ANSWER)) + audio_player_activate(sink); } } diff --git a/docs/ng_control_protocol.md b/docs/ng_control_protocol.md index 712327784..8f33b5217 100644 --- a/docs/ng_control_protocol.md +++ b/docs/ng_control_protocol.md @@ -805,6 +805,14 @@ Spaces in each string may be replaced by hyphens. the DSP to detect in-band DTMF audio tones even when it wouldn't otherwise be necessary. +* `early media` + + Used in conjunction with the audio player. If set, audio playback is + started immediately when processing an `offer` message. The default + behaviour is to start the audio player only after the `answer` has been + processed, or when any audio to be played back has actually been received + (either from another party to the call, or via the `play media` command). + * `full rtcp attribute` Include the full version of the `a=rtcp` line (complete with network address) instead of diff --git a/include/audio_player.h b/include/audio_player.h index af8bdfc08..7bf2176ec 100644 --- a/include/audio_player.h +++ b/include/audio_player.h @@ -22,6 +22,7 @@ struct rtp_payload_type; bool audio_player_setup(struct call_media *, const struct rtp_payload_type *, unsigned int size_ms, unsigned int delay_ms); +void audio_player_activate(struct call_media *); void audio_player_free(struct call_media *); void audio_player_start(struct call_media *); @@ -36,6 +37,7 @@ void audio_player_add_frame(struct audio_player *, uint32_t ssrc, AVFrame *); INLINE void audio_player_start(struct call_media *m) { } INLINE void audio_player_free(struct call_media *m) { } INLINE void audio_player_stop(struct call_media *m) { } +INLINE void audio_player_activate(struct call_media *m) { } #endif diff --git a/include/call_interfaces.h b/include/call_interfaces.h index 083bf8186..0e9beeb4e 100644 --- a/include/call_interfaces.h +++ b/include/call_interfaces.h @@ -162,6 +162,7 @@ struct sdp_ng_flags { single_codec:1, reuse_codec:1, allow_transcoding:1, + early_media:1, accept_any:1, inject_dtmf:1, detect_dtmf:1, diff --git a/lib/mix_buffer.c b/lib/mix_buffer.c index 51778e92e..cbb3d13e8 100644 --- a/lib/mix_buffer.c +++ b/lib/mix_buffer.c @@ -73,8 +73,8 @@ static void fill_up_to(struct mix_buffer *mb, unsigned int up_to) { void *mix_buffer_read_fast(struct mix_buffer *mb, unsigned int samples, unsigned int *size) { LOCK(&mb->lock); - if (samples > mb->size) { - *size = 0; // error + if (samples > mb->size || !mb->active) { + *size = 0; // error or inactive return NULL; } @@ -245,6 +245,8 @@ bool mix_buffer_write_delay(struct mix_buffer *mb, uint32_t ssrc, const void *bu if (created) mix_buff_src_shift_delay(mb, src, last, now); + mb->active = true; + // loop twice at the most to re-run logic after a reset while (true) { // shortcut if we're at the write head @@ -284,8 +286,8 @@ static struct ssrc_entry *mix_buffer_ssrc_new(void *p) { // struct must be zeroed already -bool mix_buffer_init(struct mix_buffer *mb, enum AVSampleFormat fmt, unsigned int clockrate, - unsigned int channels, unsigned int size_ms, unsigned int delay_ms) +bool mix_buffer_init_active(struct mix_buffer *mb, enum AVSampleFormat fmt, unsigned int clockrate, + unsigned int channels, unsigned int size_ms, unsigned int delay_ms, bool active) { switch (fmt) { case AV_SAMPLE_FMT_S16: @@ -305,6 +307,7 @@ bool mix_buffer_init(struct mix_buffer *mb, enum AVSampleFormat fmt, unsigned in mb->clockrate = clockrate; mb->channels = channels; mb->delay = delay; + mb->active = active; mb->ssrc_hash = create_ssrc_hash_full_fast(mix_buffer_ssrc_new, mb); diff --git a/lib/mix_buffer.h b/lib/mix_buffer.h index 8b422ab4a..9328d5732 100644 --- a/lib/mix_buffer.h +++ b/lib/mix_buffer.h @@ -50,6 +50,7 @@ struct mix_buffer { unsigned int delay; // initial write delay for new inputs/sources unsigned int loops; // how many times the write pos has circled around + bool active; // to optionally suppress early media // implementation details const struct mix_buffer_impl *impl; @@ -58,8 +59,14 @@ struct mix_buffer { }; -bool mix_buffer_init(struct mix_buffer *, enum AVSampleFormat, unsigned int clockrate, - unsigned int channels, unsigned int size_ms, unsigned int delay_ms); +bool mix_buffer_init_active(struct mix_buffer *, enum AVSampleFormat, unsigned int clockrate, + unsigned int channels, unsigned int size_ms, unsigned int delay_ms, bool active); +#define mix_buffer_init(mb, fmt, clockrate, channels, size_ms, delay_ms) \ + mix_buffer_init_active(mb, fmt, clockrate, channels, size_ms, delay_ms, true) +INLINE void mix_buffer_activate(struct mix_buffer *mb) { + LOCK(&mb->lock); + mb->active = true; +} void mix_buffer_destroy(struct mix_buffer *); void *mix_buffer_read_fast(struct mix_buffer *, unsigned int samples, unsigned int *size); diff --git a/utils/rtpengine-ng-client b/utils/rtpengine-ng-client index 70a249044..691b3bae5 100755 --- a/utils/rtpengine-ng-client +++ b/utils/rtpengine-ng-client @@ -49,6 +49,7 @@ my @flags = qw( passthrough no-passthrough pause + early-media ); my @string_opts = qw(