add stats to keepalive & change some defaults

kamailio/keepalive.cfg

@@ -2,13 +2,18 @@
 
 #!trydef KEEPALIVE_NAT_ONLY 0
 #!trydef KEEPALIVE_UDP_ONLY 0
-#!trydef KEEPALIVE_TIMERS 8
-#!trydef KEEPALIVE_INTERVAL 30
+#!trydef KEEPALIVE_TIMERS 4
+#!trydef KEEPALIVE_INTERVAL 60
 #!trydef KEEPALIVE_TIMEOUT 5000
 #!trydef KEEPALIVE_FAILED_THRESHOLD 2
 #!trydef KEEPALIVE_EXPIRE_SUBSCRIPTIONS 1
 #!trydef KEEPALIVE_EXPIRE_REGISTRATIONS 1
+#!trydef KEEPALIVE_FAILED_ACTION 1
 #!trydef KEEPALIVE_FAILED_LOG_LEVEL 0
+#!trydef KEEPALIVE_EXPIRED_SUBSCRIPTION_ACTION 1
+#!trydef KEEPALIVE_EXPIRED_REGISTRATION_ACTION 1
+#!trydef KEEPALIVE_ON_SUBSCRIPTION_ACTION 1
+#!trydef KEEPALIVE_ON_REGISTRATION_ACTION 1
 
 #!substdef "!KEEPALIVE_S_FROM_URI!sip:keepalive@MY_HOSTNAME!g"
 #!substdef "!KEEPALIVE_S_TIMERS!$def(KEEPALIVE_TIMERS)!g"
@@ -20,6 +25,12 @@ kazoo.keepalive_failed_threshold = KEEPALIVE_FAILED_THRESHOLD descr "how many ti
 kazoo.keepalive_expire_subscriptions = KEEPALIVE_EXPIRE_SUBSCRIPTIONS descr "expires subscriptions that do not respond to OPTIONS"
 kazoo.keepalive_expire_registrations = KEEPALIVE_EXPIRE_REGISTRATIONS descr "expires registrations that do not respond to OPTIONS"
 kazoo.keepalive_failed_log_level = KEEPALIVE_FAILED_LOG_LEVEL descr "loglevel for keepalive failed reply"
+kazoo.keepalive_failed_action = KEEPALIVE_FAILED_ACTION descr "action for devices that exceed the threshold. 1 = disable, 2 = delete"
+kazoo.keepalive_interval = KEEPALIVE_INTERVAL descr "interval in seconds between attempts to send OPTIONS to device"
+kazoo.keepalive_expired_registration_action = KEEPALIVE_EXPIRED_REGISTRATION_ACTION descr "action when registrar expires a registration, 1 = delete , 2 = disable, 0 = none"
+kazoo.keepalive_expired_subscription_action = KEEPALIVE_EXPIRED_SUBSCRIPTION_ACTION descr "action when presence expires a subscription, 1 = delete , 2 = disable, 0 = none"
+kazoo.keepalive_on_registration_action = KEEPALIVE_ON_REGISTRATION_ACTION descr "action on registration, 1 = insert in keepalive , 0 = none"
+kazoo.keepalive_on_subscription_action = KEEPALIVE_ON_SUBSCRIPTION_ACTION descr "action on subscription, 1 = insert in keepalive , 0 = none"
 
 modparam("rtimer", "timer", "name=keepalive_timer;interval=1;mode=KEEPALIVE_S_TIMERS;")
 modparam("rtimer", "exec", "timer=keepalive_timer;route=KEEPALIVE_TIMER")
@@ -27,67 +38,109 @@ modparam("rtimer", "exec", "timer=keepalive_timer;route=KEEPALIVE_TIMER")
 modparam("rtimer", "timer", "name=keepalive_db_timer;interval=1;mode=1;")
 modparam("rtimer", "exec", "timer=keepalive_db_timer;route=KEEPALIVE_DB_TIMER")
 
+modparam("rtimer", "timer", "name=keepalive_cleanup;interval=5;mode=1;")
+modparam("rtimer", "exec", "timer=keepalive_cleanup;route=KEEPALIVE_CLEANUP")
+
 modparam("mqueue","mqueue", "name=keepalive_db_queue")
 
+modparam("statistics","variable", "keepalive:success")
+modparam("statistics","variable", "keepalive:failure")
+modparam("statistics","variable", "keepalive:db:success")
+modparam("statistics","variable", "keepalive:db:failure")
+modparam("statistics","variable", "keepalive:client_options")
+modparam("statistics","variable", "keepalive:client_notify")
+modparam("statistics","variable", "keepalive:disabled")
+modparam("statistics","variable", "keepalive:removed")
+modparam("statistics","variable", "keepalive:expired_registrations")
+modparam("statistics","variable", "keepalive:expired_subscriptions")
+
+modparam("statistics","variable", "keepalive:from_registration")
+modparam("statistics","variable", "keepalive:from_subscription")
+
+modparam("statistics","variable", "keepalive:removed_from_registration")
+modparam("statistics","variable", "keepalive:removed_from_subscription")
+
+modparam("statistics","variable", "keepalive:disabled_from_expired_registration")
+modparam("statistics","variable", "keepalive:removed_from_expired_registration")
+
+modparam("statistics","variable", "keepalive:disabled_from_expired_subscription")
+modparam("statistics","variable", "keepalive:removed_from_expired_subscription")
+
 route[KEEPALIVE_DB_TIMER]
 {
   $var(runloop) = 1;
   while(mq_fetch("keepalive_db_queue") == 1 && $var(runloop) < MAX_WHILE_LOOPS) {
      $var(ci) = $mqk(keepalive_db_queue);
-     xlog("L_DEBUG", "Query : $mqv(keepalive_db_queue)\n");
-     lock("keepalive");
+     xlog("L_DEBUG", "Query : $var(ci) => $mqv(keepalive_db_queue)\n");
      $var(sqlres) = sql_query("cb", "$mqv(keepalive_db_queue)");
      xlog("L_DEBUG", "Query result : $var(sqlres)\n");
      if($var(sqlres) < 0) {
         xlog("L_ERROR", "$var(ci)|log|error running query : $mqv(keepalive_db_queue)\n");
      } else {
+       $var(stat_update) = $_s(+$sqlrows(cb));
+       update_stat("$var(ci)", "$var(stat_update)");
+     
        $var(nrows) = $sqlrows(cb);
-       xlog("L_DEBUG", "$var(ci)|log|end UPDATED $var(nrows)\n");
+       xlog("L_DEBUG", "$var(ci)|log|end UPDATED $var(nrows) => $var(stat_update)\n");
        if($var(nrows) == 0) {
            xlog("L_DEBUG", "$var(ci)|log|error no rows affected when running query\n");
        }
      }
-     unlock("keepalive");
      $var(runloop) = $var(runloop) + 1;
   }
 }
 
 route[KEEPALIVE_CLEANUP]
 {
-   lock("keepalive");
+   $var(Query) = $_s(UPDATE keepalive SET SELECTED = 9 where selected < 3 and failed > $sel(cfg_get.kazoo.keepalive_failed_threshold));
+   sql_query("cb", "$var(Query)");
+
    if($sel(cfg_get.kazoo.keepalive_expire_registrations) == 1) {
-       $var(Query) = $_s(update location set expires = last_modified where id in(select b.id from w_keepalive_contact a inner join w_location_contact b on a.contact = b.contact where slot = $rtimer_worker AND failed > $sel(cfg_get.kazoo.keepalive_failed_threshold)));
+       $var(Query) = $_s(update location set expires = last_modified where id in(select b.id from w_keepalive_contact a inner join w_location_contact b on a.contact = b.contact where selected = 9));
        sql_query("cb", "$var(Query)");
+       $var(stat_update) = $_s(+$sqlrows(cb));
+       update_stat("keepalive:expired_registrations", "$var(stat_update)");
    }
+
    if($sel(cfg_get.kazoo.keepalive_expire_subscriptions) == 1) {
-       $var(Query) = $_s(DELETE FROM active_watchers where id in(select b.id from w_keepalive_contact a inner join w_watchers_contact b on a.contact = b.contact where slot = $rtimer_worker and failed > $sel(cfg_get.kazoo.keepalive_failed_threshold)));
+       $var(Query) = $_s(DELETE FROM active_watchers where id in(select b.id from w_keepalive_contact a inner join w_watchers_contact b on a.contact = b.contact where selected = 9));
        sql_query("cb", "$var(Query)");
+       $var(stat_update) = $_s(+$sqlrows(cb));
+       update_stat("keepalive:expired_subscriptions", "$var(stat_update)");
+   }
+   
+   if($sel(cfg_get.kazoo.keepalive_failed_action) == 2) {
+       ## disable
+       $var(Query) = $_s(UPDATE keepalive SET SELECTED = 10 where selected = 9);
+       $var(stat) = "keepalive:disabled";
+   } else if($sel(cfg_get.kazoo.keepalive_failed_action) == 1) {
+       ## delete - will be recreated on registration/subscription with same contact
+       $var(Query) = $_s(DELETE FROM keepalive where selected = 9);
+       $var(stat) = "keepalive:removed";
    }
-   $var(Query) = $_s(DELETE FROM keepalive where slot = $rtimer_worker and failed > $sel(cfg_get.kazoo.keepalive_failed_threshold));
    sql_query("cb", "$var(Query)");
-   unlock("keepalive");
+   $var(stat_update) = $_s(+$sqlrows(cb));
+   update_stat("$var(stat)", "$var(stat_update)");
 }
 
 route[KEEPALIVE_TIMER]
 {
-   route(KEEPALIVE_CLEANUP);
-
-   lock("keepalive");
-   $var(Query) = $_s(UPDATE keepalive SET selected = 1 WHERE slot = $rtimer_worker AND selected = 0 AND time_sent < datetime('now', '-$def(KEEPALIVE_INTERVAL) seconds'));
+   $var(base_slot) = $rtimer_worker * $sel(cfg_get.kazoo.keepalive_interval);
+   $var(slot) = $var(base_slot) + $var(tick);
+   $var(Query) = $_s(UPDATE keepalive SET selected = 1 WHERE slot = $var(slot) AND selected = 0 AND time_sent < datetime('now', '-$sel(cfg_get.kazoo.keepalive_interval) seconds'));
    $var(sqlres) = sql_query("cb", "$var(Query)");
    if($var(sqlres) < 0) {
-       xlog("L_ERROR", "$rtimer_worker|log|error running query : $var(Query)\n");
+       xlog("L_ERROR", "$rtimer_worker|$var(tick)|log|error running query : $var(Query)\n");
    } else {
        $var(nrows) = $sqlrows(cb);
-       xlog("L_DEBUG", "$rtimer_worker|log|selected $var(nrows) endpoints to ping\n");
+       xlog("L_DEBUG", "$rtimer_worker|$var(tick)|log|selected $var(nrows) endpoints to ping\n");
    }
-   unlock("keepalive");
    
-   lock("keepalive");   
-   $var(Query) = $_s(SELECT id, contact, sockinfo from keepalive WHERE slot = $rtimer_worker AND selected = 1);
-   xlog("L_DEBUG", "$rtimer_worker|timer|SQL => $var(Query)\n");
-   if (sql_xquery("cb", "$var(Query)", "ra") == 1)
-   {
+   $var(Query) = $_s(SELECT id, contact, sockinfo from keepalive WHERE slot = $var(slot) AND selected = 1);
+   xlog("L_DEBUG", "$rtimer_worker|$var(tick)|timer|SQL => $var(Query)\n");
+   $var(result) =sql_xquery("cb", "$var(Query)", "ra");
+   
+   if($var(result) == 1) {
         while($xavp(ra) != $null) {
             $var(loop) = 0;
             while($xavp(ra) != $null && $var(loop) < MAX_WHILE_LOOPS) {
@@ -97,17 +150,18 @@ route[KEEPALIVE_TIMER]
             }
         }
    }
-   unlock("keepalive");
    
-   lock("keepalive");
-   $var(Query) = $_s(UPDATE keepalive SET selected = 2 WHERE slot = $rtimer_worker AND selected = 1);
+   $var(Query) = $_s(UPDATE keepalive SET selected = 2 WHERE slot = $var(slot) AND selected = 1);
    $var(sqlres) = sql_query("cb", "$var(Query)");
    if($var(sqlres) < 0) {
-       xlog("L_ERROR", "$rtimer_worker|log|error running query : $var(Query)\n");
+       xlog("L_ERROR", "$rtimer_worker|$var(tick)|log|error running query : $var(Query)\n");
    }
-   unlock("keepalive");
-   
    
+   $var(tick) = $var(tick) + 1;
+   if($var(tick) > $sel(cfg_get.kazoo.keepalive_interval))  {
+       $var(tick) = 0;
+   } 
+
 }
 
 route[KEEPALIVE_SEND_PING]
@@ -129,19 +183,21 @@ route[KEEPALIVE_SEND_PING]
 onreply_route[KEEPALIVE_REPLY]
 {
    xlog("L_DEBUG", "$ci|keepalive|KEEPALIVE REPLY $(tu{nameaddr.uri})\n");
-   $var(Query) = $_s(UPDATE keepalive SET selected = 0, failed = 0, time_sent = datetime('now') WHERE id = $(fu{uri.param,nat_id}););
+   $var(Query) = $_s(UPDATE keepalive SET selected = 0, failed = 0, time_sent = datetime('now') WHERE id = $(fu{uri.param,nat_id}) AND SELECTED = 2);
    xlog("L_DEBUG", "$ci|keepalive|KEEPALIVE UPDATE SQL => '$var(Query)'\n");
-   mq_add("keepalive_db_queue", "$uuid(g)", "$var(Query)");
-   t_drop();
+   mq_add("keepalive_db_queue", "keepalive:db:success", "$var(Query)");
+   update_stat("keepalive:success", "+1");
+   resetflag(FLAG_SIP_TRACE);
 }
 
 failure_route[KEEPALIVE_FAULT]
 {
    xlog("$(sel(cfg_get.kazoo.keepalive_failed_log_level){s.int})", "$ci|keepalive|received error $T_reply_code $T_reply_reason from $(tu{nameaddr.uri})\n");
-   $var(Query) = $_s(UPDATE keepalive SET selected = 0, failed = failed + 1, time_sent = datetime('now') WHERE id = $(fu{uri.param,nat_id}););
+   $var(Query) = $_s(UPDATE keepalive SET selected = 0, failed = failed + 1, time_sent = datetime('now') WHERE id = $(fu{uri.param,nat_id}) AND SELECTED = 2);
    xlog("L_DEBUG", "$ci|keepalive|KEEPALIVE REMOVE SQL => '$var(Query)'\n");
-   mq_add("keepalive_db_queue", "$uuid(g)", "$var(Query)");
-   t_drop();
+   mq_add("keepalive_db_queue", "keepalive:db:failure", "$var(Query)");
+   update_stat("keepalive:failure", "+1");
+   resetflag(FLAG_SIP_TRACE);
 }
 
 route[KEEPALIVE_PING]
@@ -163,6 +219,10 @@ route[KEEPALIVE_PING]
 
 route[KEEPALIVE_ON_REGISTRATION]
 {
+
+   if($sel(cfg_get.kazoo.keepalive_on_registration_action) == 0) {
+      return;
+   }
    
    if($proto == "ws" || $proto == "wss") {
       return;
@@ -185,41 +245,57 @@ route[KEEPALIVE_ON_REGISTRATION]
 
    if($var(save_result) == 3) {
         $var(sql) = $_s(DELETE FROM keepalive WHERE contact = "$var(contact)");
+        $var(stat) = "keepalive:removed_from_registration";
    } else {
-        $var(slot) = $(var(contact){s.corehash, KEEPALIVE_S_TIMERS});
-        $var(sql) = $_s(INSERT OR IGNORE INTO keepalive (contact, sockinfo, slot) values("$var(contact)", "$(RAut{uri.tosocket})", $var(slot)));
+        $var(max_slots) = $sel(cfg_get.kazoo.keepalive_interval) * KEEPALIVE_S_TIMERS;
+        $var(slot) = $(var(contact){s.corehash, $var(max_slots)});
+        $var(sql) = $_s(INSERT OR IGNORE INTO keepalive (contact, received, sockinfo, slot) values("$var(contact)", "$var(alias)", "$(RAut{uri.tosocket})", $var(slot)));
+        $var(stat) = "keepalive:from_registration";
    }
-   mq_add("keepalive_db_queue", "$ci", "$var(sql)");
+   mq_add("keepalive_db_queue", "$var(stat)", "$var(sql)");
 
    return;
 }
 
 route[KEEPALIVE_ON_SUBSCRIBE]
 {
+   if($sel(cfg_get.kazoo.keepalive_on_subscription_action) == 0) {
+      return;
+   }
+
    if(!( ($sel(cfg_get.kazoo.keepalive_udp_only) == 1 && $proto != "udp")
       || (!isflagset(FLT_NATS) && $sel(cfg_get.kazoo.keepalive_nat_only) == 1)
       || ($proto == "ws" || $proto == "wss")
      )) {
-     $var(slot) = $(subs(contact){s.corehash, KEEPALIVE_S_TIMERS});
-     $var(sql) = $_s(INSERT OR IGNORE INTO keepalive (contact, sockinfo, slot) values("$subs(contact)", "$subs(sockinfo)", $var(slot)));
-     mq_add("keepalive_db_queue", "$ci", "$var(sql)");
+     $var(max_slots) = $sel(cfg_get.kazoo.keepalive_interval) * KEEPALIVE_S_TIMERS;
+     $var(slot) = $(subs(contact){s.corehash, $var(max_slots)});
+     $var(alias) = $(subs(contact){uri.param,alias});
+     $var(sql) = $_s(INSERT OR IGNORE INTO keepalive (contact, received, sockinfo, slot) values("$subs(contact)", "$var(alias)", "$subs(sockinfo)", $var(slot)));
+     mq_add("keepalive_db_queue", "keepalive:from_subscription", "$var(sql)");
    }
 }
 
 route[KEEPALIVE_ON_EXPIRED_REGISTRATION]
 {
-   $var(Query) = $_s(DELETE FROM keepalive where contact like "$ulc(exp=>addr)%");
-   mq_add("keepalive_db_queue", "$uuid(g)", "$var(Query)");
+   if($sel(cfg_get.kazoo.keepalive_expired_registration_action) == 2) {
+       ## disable
+       $var(Query) = $_s(UPDATE  keepalive SET SELECTED = 10 where selected < 3 and contact like "$ulc(exp=>addr)%");
+       mq_add("keepalive_db_queue", "keepalive:disabled_from_expired_registration", "$var(Query)");
+   } else if($sel(cfg_get.kazoo.keepalive_expired_registration_action) == 1) {
+       ## delete - will be recreated on registration with same contact
+       $var(Query) = $_s(DELETE FROM keepalive where selected < 3 and contact like "$ulc(exp=>addr)%");
+       mq_add("keepalive_db_queue", "keepalive:removed_from_expired_registration", "$var(Query)");
+   }
 }
 
 route[KEEPALIVE_ON_OPTIONS]
 {
-   $var(Query) = $_s(UPDATE keepalive set selected = 3 where contact like "%alias=$si~$sp~$prid%");
-   mq_add("keepalive_db_queue", "$uuid(g)", "$var(Query)");
+   $var(Query) = $_s(UPDATE keepalive set selected = 3 where received = "$si~$sp~$prid" and selected <> 3 );
+   mq_add("keepalive_db_queue", "keepalive:client_options", "$var(Query)");
 }
 
 route[KEEPALIVE_ON_NOTIFY]
 {
-   $var(Query) = $_s(UPDATE keepalive set selected = 4 where contact like "%alias=$si~$sp~$prid%");
-   mq_add("keepalive_db_queue", "$uuid(g)", "$var(Query)");
+   $var(Query) = $_s(UPDATE keepalive set selected = 4 where received = "$si~$sp~$prid" and selected <> 4 );
+   mq_add("keepalive_db_queue", "keepalive:client_notify", "$var(Query)");
 }