From 074acd0f0a412ae657a5841417cde6c0165cac54 Mon Sep 17 00:00:00 2001
From: Richard Fuchs <rfuchs@sipwise.com>
Date: Thu, 8 Feb 2024 10:49:44 -0500
Subject: [PATCH] MT#55283 optimise timer tree lookup

Look up the first object in the tree to determine the needed sleep time.
But instead of leaving the object in the tree for the next (post-sleep)
iteration, which requires another tree lookup, remove it from the tree
and remember it, anticipating that nothing would get added into the tree
to be scheduled sooner. This saves us from having to do another tree
lookup for each timer that runs.

Adapt the scheduling function to be able to handle this and wake up the
thread sooner if necessary.

Change-Id: I9297346bc4aa8d2f68ecb833be8f71ce62a3bbfe
---
 daemon/timerthread.c  | 57 ++++++++++++++++++++++++++++++++-----------
 include/timerthread.h |  1 +
 2 files changed, 44 insertions(+), 14 deletions(-)

diff --git a/daemon/timerthread.c b/daemon/timerthread.c
index 609a8b5b0..312f47c60 100644
--- a/daemon/timerthread.c
+++ b/daemon/timerthread.c
@@ -15,6 +15,7 @@ static void timerthread_thread_init(struct timerthread_thread *tt, struct timert
 	cond_init(&tt->cond);
 	tt->parent = parent;
 	ZERO(tt->next_wake);
+	tt->obj = NULL;
 }
 
 void timerthread_init(struct timerthread *tt, unsigned int num, void (*func)(void *)) {
@@ -35,6 +36,8 @@ static int __tt_put_all(void *k, void *d, void *p) {
 static void timerthread_thread_destroy(struct timerthread_thread *tt) {
 	g_tree_foreach(tt->tree, __tt_put_all, tt);
 	g_tree_destroy(tt->tree);
+	if (tt->obj)
+		obj_put(tt->obj);
 	mutex_destroy(&tt->lock);
 }
 
@@ -56,24 +59,33 @@ static void timerthread_run(void *p) {
 	while (!rtpe_shutdown) {
 		gettimeofday(&rtpe_now, NULL);
 
-		/* lock our list and get the first element */
-		struct timerthread_obj *tt_obj = g_tree_find_first(tt->tree, NULL, NULL);
-		/* scheduled to run? if not, we just go to sleep, otherwise we remove it from the tree,
-		 * steal the reference and run it */
 		long long sleeptime = 10000000;
-		if (!tt_obj)
-			goto sleep;
+		// find the first element if we haven't determined it yet
+		struct timerthread_obj *tt_obj = tt->obj;
+		if (!tt_obj) {
+			tt_obj = g_tree_find_first(tt->tree, NULL, NULL);
+			if (!tt_obj)
+				goto sleep_now;
+
+			// immediately steal reference
+			// XXX ideally we would have a tree_steal_first() function
+			g_tree_remove(tt->tree, tt_obj);
+		}
+
+		// scheduled to run? if not, then we remember this object/reference and go to sleep
 		sleeptime = timeval_diff(&tt_obj->next_check, &rtpe_now);
-		if (sleeptime > 0)
+
+		if (sleeptime > 0) {
+			tt->obj = tt_obj;
 			goto sleep;
+		}
 
-		// steal reference
-		g_tree_remove(tt->tree, tt_obj);
 		// pretend we're running exactly at the scheduled time
 		rtpe_now = tt_obj->next_check;
 		ZERO(tt_obj->next_check);
 		tt_obj->last_run = rtpe_now;
 		ZERO(tt->next_wake);
+		tt->obj = NULL;
 		mutex_unlock(&tt->lock);
 
 		// run and release
@@ -85,9 +97,10 @@ static void timerthread_run(void *p) {
 		mutex_lock(&tt->lock);
 		continue;
 
-sleep:;
+sleep:
 		/* figure out how long we should sleep */
-		sleeptime = MIN(10000000, sleeptime); /* 100 ms at the most */
+		sleeptime = MIN(10000000, sleeptime);
+sleep_now:;
 		struct timeval tv = rtpe_now;
 		timeval_add_usec(&tv, sleeptime);
 		tt->next_wake = tv;
@@ -113,13 +126,23 @@ void timerthread_obj_schedule_abs_nl(struct timerthread_obj *tt_obj, const struc
 
 	if (tt_obj->next_check.tv_sec && timeval_cmp(&tt_obj->next_check, tv) <= 0)
 		return; /* already scheduled sooner */
-	if (!g_tree_remove(tt->tree, tt_obj))
-		obj_hold(tt_obj); /* if it wasn't removed, we make a new reference */
+	if (!g_tree_remove(tt->tree, tt_obj)) {
+		if (tt->obj == tt_obj)
+			tt->obj = NULL;
+		else
+			obj_hold(tt_obj); /* if it wasn't removed, we make a new reference */
+	}
 	tt_obj->next_check = *tv;
 	g_tree_insert(tt->tree, tt_obj, tt_obj);
 	// need to wake the thread?
-	if (tt->next_wake.tv_sec && timeval_cmp(tv, &tt->next_wake) < 0)
+	if (tt->next_wake.tv_sec && timeval_cmp(tv, &tt->next_wake) < 0) {
+		// make sure we can get picked first: move pre-picked object back into tree
+		if (tt->obj && tt->obj != tt_obj) {
+			g_tree_insert(tt->tree, tt->obj, tt->obj);
+			tt->obj = NULL;
+		}
 		cond_signal(&tt->cond);
+	}
 }
 
 void timerthread_obj_deschedule(struct timerthread_obj *tt_obj) {
@@ -134,6 +157,12 @@ void timerthread_obj_deschedule(struct timerthread_obj *tt_obj) {
 	if (!tt_obj->next_check.tv_sec)
 		goto nope; /* already descheduled */
 	gboolean ret = g_tree_remove(tt->tree, tt_obj);
+	if (!ret) {
+		if (tt->obj == tt_obj) {
+			tt->obj = NULL;
+			ret = TRUE;
+		}
+	}
 	ZERO(tt_obj->next_check);
 	if (ret)
 		obj_put(tt_obj);
diff --git a/include/timerthread.h b/include/timerthread.h
index 58b65a34b..7c0e3de3d 100644
--- a/include/timerthread.h
+++ b/include/timerthread.h
@@ -15,6 +15,7 @@ struct timerthread_thread {
 	mutex_t lock;
 	cond_t cond;
 	struct timeval next_wake;
+	struct timerthread_obj *obj;
 };
 
 struct timerthread {