From 482b004ef59f2bc48389c28b49b0e803c545e9a0 Mon Sep 17 00:00:00 2001 From: Lucian Balaceanu Date: Wed, 30 Sep 2015 09:28:45 +0200 Subject: [PATCH] CLI output for Graphite statistics --- daemon/call.c | 1 + daemon/call.h | 11 ++++--- daemon/cli.c | 22 +++++++++++-- daemon/graphite.c | 81 +++++++++++++++++++++++++++-------------------- 4 files changed, 74 insertions(+), 41 deletions(-) diff --git a/daemon/call.c b/daemon/call.c index 308689941..6997bff7f 100644 --- a/daemon/call.c +++ b/daemon/call.c @@ -1529,6 +1529,7 @@ struct callmaster *callmaster_new(struct poller *p) { c->totalstats_interval.managed_sess_min = 0; c->totalstats_interval.managed_sess_max = 0; + mutex_init(&c->totalstats_lastinterval_lock); mutex_init(&c->cngs_lock); c->cngs_hash = g_hash_table_new(in6_addr_hash, in6_addr_eq); diff --git a/daemon/call.h b/daemon/call.h index 53fef6156..e1fcab42b 100644 --- a/daemon/call.h +++ b/daemon/call.h @@ -463,20 +463,23 @@ struct callmaster { /* XXX rework these */ struct stats statsps; /* per second stats, running timer */ struct stats stats; /* copied from statsps once a second */ - struct totalstats totalstats; - struct totalstats totalstats_interval; + struct totalstats totalstats; + struct totalstats totalstats_interval; + mutex_t totalstats_lastinterval_lock; + struct totalstats totalstats_lastinterval; + /* control_ng_stats stuff */ mutex_t cngs_lock; GHashTable *cngs_hash; - struct poller *poller; + struct poller *poller; pcre *info_re; pcre_extra *info_ree; pcre *streams_re; pcre_extra *streams_ree; struct callmaster_config conf; - struct timeval latest_graphite_interval_start; + struct timeval latest_graphite_interval_start; }; struct call_stats { diff --git a/daemon/cli.c b/daemon/cli.c index 55813c286..f59d1ac47 100644 --- a/daemon/cli.c +++ b/daemon/cli.c @@ -28,8 +28,8 @@ static const char* TRUNCATED = " ... Output truncated. Increase Output Buffer static void cli_incoming_list_totals(char* buffer, int len, struct callmaster* m, char* replybuffer, const char* outbufend) { int printlen=0; - struct timeval avg; - u_int64_t num_sessions; + struct timeval avg, calls_dur_iv; + u_int64_t num_sessions, min_sess_iv, max_sess_iv; mutex_lock(&m->totalstats.total_average_lock); avg = m->totalstats.total_average_call_dur; @@ -94,6 +94,24 @@ static void cli_incoming_list_totals(char* buffer, int len, struct callmaster* m ADJUSTLEN(printlen,outbufend,replybuffer); mutex_unlock(&m->cngs_lock); g_list_free(list); + + mutex_lock(&m->totalstats_lastinterval_lock); + calls_dur_iv = m->totalstats_lastinterval.total_calls_duration_interval; + min_sess_iv = m->totalstats_lastinterval.managed_sess_min; + max_sess_iv = m->totalstats_lastinterval.managed_sess_max; + mutex_unlock(&m->totalstats_lastinterval_lock); + + printlen = snprintf(replybuffer,(outbufend-replybuffer), "\nGraphite interval statistics (last reported values to graphite):\n"); + ADJUSTLEN(printlen,outbufend,replybuffer); + printlen = snprintf(replybuffer,(outbufend-replybuffer), " Total calls duration :%ld.%06ld\n\n",calls_dur_iv.tv_sec,calls_dur_iv.tv_usec); + ADJUSTLEN(printlen,outbufend,replybuffer); + printlen = snprintf(replybuffer,(outbufend-replybuffer), " Min managed sessions :"UINT64F"\n", min_sess_iv); + ADJUSTLEN(printlen,outbufend,replybuffer); + printlen = snprintf(replybuffer,(outbufend-replybuffer), " Max managed sessions :"UINT64F"\n", max_sess_iv); + ADJUSTLEN(printlen,outbufend,replybuffer); + + printlen = snprintf(replybuffer,(outbufend-replybuffer), "\n\n"); + ADJUSTLEN(printlen,outbufend,replybuffer); } static void cli_incoming_list_callid(char* buffer, int len, struct callmaster* m, char* replybuffer, const char* outbufend) { diff --git a/daemon/graphite.c b/daemon/graphite.c index 8993f1ccc..69d5b0b4a 100644 --- a/daemon/graphite.c +++ b/daemon/graphite.c @@ -29,6 +29,7 @@ static time_t next_run; // HEAD: static time_t g_now, next_run; static char* graphite_prefix = NULL; static struct timeval graphite_interval_tv; +static struct totalstats graphite_stats; void set_graphite_interval_tv(struct timeval *tv) { graphite_interval_tv = *tv; @@ -92,7 +93,7 @@ error: return -1; } -int send_graphite_data() { +int send_graphite_data(struct totalstats *sent_data) { int rc=0; @@ -111,28 +112,29 @@ int send_graphite_data() { char data_to_send[8192]; char* ptr = data_to_send; - struct totalstats ts; + + struct totalstats *ts = sent_data; /* atomically copy values to stack and reset to zero */ - atomic64_local_copy_zero_struct(&ts, &cm->totalstats_interval, total_timeout_sess); - atomic64_local_copy_zero_struct(&ts, &cm->totalstats_interval, total_rejected_sess); - atomic64_local_copy_zero_struct(&ts, &cm->totalstats_interval, total_silent_timeout_sess); - atomic64_local_copy_zero_struct(&ts, &cm->totalstats_interval, total_regular_term_sess); - atomic64_local_copy_zero_struct(&ts, &cm->totalstats_interval, total_forced_term_sess); - atomic64_local_copy_zero_struct(&ts, &cm->totalstats_interval, total_relayed_packets); - atomic64_local_copy_zero_struct(&ts, &cm->totalstats_interval, total_relayed_errors); - atomic64_local_copy_zero_struct(&ts, &cm->totalstats_interval, total_nopacket_relayed_sess); - atomic64_local_copy_zero_struct(&ts, &cm->totalstats_interval, total_oneway_stream_sess); + atomic64_local_copy_zero_struct(ts, &cm->totalstats_interval, total_timeout_sess); + atomic64_local_copy_zero_struct(ts, &cm->totalstats_interval, total_rejected_sess); + atomic64_local_copy_zero_struct(ts, &cm->totalstats_interval, total_silent_timeout_sess); + atomic64_local_copy_zero_struct(ts, &cm->totalstats_interval, total_regular_term_sess); + atomic64_local_copy_zero_struct(ts, &cm->totalstats_interval, total_forced_term_sess); + atomic64_local_copy_zero_struct(ts, &cm->totalstats_interval, total_relayed_packets); + atomic64_local_copy_zero_struct(ts, &cm->totalstats_interval, total_relayed_errors); + atomic64_local_copy_zero_struct(ts, &cm->totalstats_interval, total_nopacket_relayed_sess); + atomic64_local_copy_zero_struct(ts, &cm->totalstats_interval, total_oneway_stream_sess); mutex_lock(&cm->totalstats_interval.total_average_lock); - ts.total_average_call_dur = cm->totalstats_interval.total_average_call_dur; - ts.total_managed_sess = cm->totalstats_interval.total_managed_sess; + ts->total_average_call_dur = cm->totalstats_interval.total_average_call_dur; + ts->total_managed_sess = cm->totalstats_interval.total_managed_sess; ZERO(cm->totalstats_interval.total_average_call_dur); ZERO(cm->totalstats_interval.total_managed_sess); mutex_unlock(&cm->totalstats_interval.total_average_lock); mutex_lock(&cm->totalstats_interval.total_calls_duration_lock); - ts.total_calls_duration_interval = cm->totalstats_interval.total_calls_duration_interval; + ts->total_calls_duration_interval = cm->totalstats_interval.total_calls_duration_interval; cm->totalstats_interval.total_calls_duration_interval.tv_sec = 0; cm->totalstats_interval.total_calls_duration_interval.tv_usec = 0; @@ -141,47 +143,47 @@ int send_graphite_data() { rwlock_lock_r(&cm->hashlock); mutex_lock(&cm->totalstats_interval.managed_sess_lock); - ts.managed_sess_max = cm->totalstats_interval.managed_sess_max; - ts.managed_sess_min = cm->totalstats_interval.managed_sess_min; + ts->managed_sess_max = cm->totalstats_interval.managed_sess_max; + ts->managed_sess_min = cm->totalstats_interval.managed_sess_min; cm->totalstats_interval.managed_sess_max = cm->totalstats.managed_sess_crt; cm->totalstats_interval.managed_sess_min = cm->totalstats.managed_sess_crt; mutex_unlock(&cm->totalstats_interval.managed_sess_lock); rwlock_unlock_r(&cm->hashlock); if (graphite_prefix!=NULL) { rc = sprintf(ptr,"%s.",graphite_prefix); ptr += rc; } - rc = sprintf(ptr, "%s.totals.call_dur %llu.%06llu %llu\n",hostname,(unsigned long long)ts.total_calls_duration_interval.tv_sec,(unsigned long long)ts.total_calls_duration_interval.tv_usec,(unsigned long long)g_now.tv_sec); ptr += rc; + rc = sprintf(ptr, "%s.totals.call_dur %llu.%06llu %llu\n",hostname,(unsigned long long)ts->total_calls_duration_interval.tv_sec,(unsigned long long)ts->total_calls_duration_interval.tv_usec,(unsigned long long)g_now.tv_sec); ptr += rc; if (graphite_prefix!=NULL) { rc = sprintf(ptr,"%s.",graphite_prefix); ptr += rc; } - rc = sprintf(ptr,"%s.totals.average_call_dur %llu.%06llu %llu\n",hostname,(unsigned long long)ts.total_average_call_dur.tv_sec,(unsigned long long)ts.total_average_call_dur.tv_usec,(unsigned long long)g_now.tv_sec); ptr += rc; + rc = sprintf(ptr,"%s.totals.average_call_dur %llu.%06llu %llu\n",hostname,(unsigned long long)ts->total_average_call_dur.tv_sec,(unsigned long long)ts->total_average_call_dur.tv_usec,(unsigned long long)g_now.tv_sec); ptr += rc; if (graphite_prefix!=NULL) { rc = sprintf(ptr,"%s.",graphite_prefix); ptr += rc; } - rc = sprintf(ptr,"%s.totals.forced_term_sess "UINT64F" %llu\n",hostname, atomic64_get_na(&ts.total_forced_term_sess),(unsigned long long)g_now.tv_sec); ptr += rc; + rc = sprintf(ptr,"%s.totals.forced_term_sess "UINT64F" %llu\n",hostname, atomic64_get_na(&ts->total_forced_term_sess),(unsigned long long)g_now.tv_sec); ptr += rc; if (graphite_prefix!=NULL) { rc = sprintf(ptr,"%s.",graphite_prefix); ptr += rc; } - rc = sprintf(ptr,"%s.totals.managed_sess "UINT64F" %llu\n",hostname, ts.total_managed_sess,(unsigned long long)g_now.tv_sec); ptr += rc; + rc = sprintf(ptr,"%s.totals.managed_sess "UINT64F" %llu\n",hostname, ts->total_managed_sess,(unsigned long long)g_now.tv_sec); ptr += rc; if (graphite_prefix!=NULL) { rc = sprintf(ptr,"%s.",graphite_prefix); ptr += rc; } - rc = sprintf(ptr,"%s.totals.managed_sess_min "UINT64F" %llu\n",hostname, ts.managed_sess_min,(unsigned long long)g_now.tv_sec); ptr += rc; + rc = sprintf(ptr,"%s.totals.managed_sess_min "UINT64F" %llu\n",hostname, ts->managed_sess_min,(unsigned long long)g_now.tv_sec); ptr += rc; if (graphite_prefix!=NULL) { rc = sprintf(ptr,"%s.",graphite_prefix); ptr += rc; } - rc = sprintf(ptr,"%s.totals.managed_sess_max "UINT64F" %llu\n",hostname, ts.managed_sess_max,(unsigned long long)g_now.tv_sec); ptr += rc; + rc = sprintf(ptr,"%s.totals.managed_sess_max "UINT64F" %llu\n",hostname, ts->managed_sess_max,(unsigned long long)g_now.tv_sec); ptr += rc; if (graphite_prefix!=NULL) { rc = sprintf(ptr,"%s.",graphite_prefix); ptr += rc; } - rc = sprintf(ptr,"%s.totals.nopacket_relayed_sess "UINT64F" %llu\n",hostname, atomic64_get_na(&ts.total_nopacket_relayed_sess),(unsigned long long)g_now.tv_sec); ptr += rc; + rc = sprintf(ptr,"%s.totals.nopacket_relayed_sess "UINT64F" %llu\n",hostname, atomic64_get_na(&ts->total_nopacket_relayed_sess),(unsigned long long)g_now.tv_sec); ptr += rc; if (graphite_prefix!=NULL) { rc = sprintf(ptr,"%s.",graphite_prefix); ptr += rc; } - rc = sprintf(ptr,"%s.totals.oneway_stream_sess "UINT64F" %llu\n",hostname, atomic64_get_na(&ts.total_oneway_stream_sess),(unsigned long long)g_now.tv_sec); ptr += rc; + rc = sprintf(ptr,"%s.totals.oneway_stream_sess "UINT64F" %llu\n",hostname, atomic64_get_na(&ts->total_oneway_stream_sess),(unsigned long long)g_now.tv_sec); ptr += rc; if (graphite_prefix!=NULL) { rc = sprintf(ptr,"%s.",graphite_prefix); ptr += rc; } - rc = sprintf(ptr,"%s.totals.regular_term_sess "UINT64F" %llu\n",hostname, atomic64_get_na(&ts.total_regular_term_sess),(unsigned long long)g_now.tv_sec); ptr += rc; + rc = sprintf(ptr,"%s.totals.regular_term_sess "UINT64F" %llu\n",hostname, atomic64_get_na(&ts->total_regular_term_sess),(unsigned long long)g_now.tv_sec); ptr += rc; if (graphite_prefix!=NULL) { rc = sprintf(ptr,"%s.",graphite_prefix); ptr += rc; } - rc = sprintf(ptr,"%s.totals.relayed_errors "UINT64F" %llu\n",hostname, atomic64_get_na(&ts.total_relayed_errors),(unsigned long long)g_now.tv_sec); ptr += rc; + rc = sprintf(ptr,"%s.totals.relayed_errors "UINT64F" %llu\n",hostname, atomic64_get_na(&ts->total_relayed_errors),(unsigned long long)g_now.tv_sec); ptr += rc; if (graphite_prefix!=NULL) { rc = sprintf(ptr,"%s.",graphite_prefix); ptr += rc; } - rc = sprintf(ptr,"%s.totals.relayed_packets "UINT64F" %llu\n",hostname, atomic64_get_na(&ts.total_relayed_packets),(unsigned long long)g_now.tv_sec); ptr += rc; + rc = sprintf(ptr,"%s.totals.relayed_packets "UINT64F" %llu\n",hostname, atomic64_get_na(&ts->total_relayed_packets),(unsigned long long)g_now.tv_sec); ptr += rc; if (graphite_prefix!=NULL) { rc = sprintf(ptr,"%s.",graphite_prefix); ptr += rc; } - rc = sprintf(ptr,"%s.totals.silent_timeout_sess "UINT64F" %llu\n",hostname, atomic64_get_na(&ts.total_silent_timeout_sess),(unsigned long long)g_now.tv_sec); ptr += rc; + rc = sprintf(ptr,"%s.totals.silent_timeout_sess "UINT64F" %llu\n",hostname, atomic64_get_na(&ts->total_silent_timeout_sess),(unsigned long long)g_now.tv_sec); ptr += rc; if (graphite_prefix!=NULL) { rc = sprintf(ptr,"%s.",graphite_prefix); ptr += rc; } - rc = sprintf(ptr,"%s.totals.timeout_sess "UINT64F" %llu\n",hostname, atomic64_get_na(&ts.total_timeout_sess),(unsigned long long)g_now.tv_sec); ptr += rc; + rc = sprintf(ptr,"%s.totals.timeout_sess "UINT64F" %llu\n",hostname, atomic64_get_na(&ts->total_timeout_sess),(unsigned long long)g_now.tv_sec); ptr += rc; if (graphite_prefix!=NULL) { rc = sprintf(ptr,"%s.",graphite_prefix); ptr += rc; } - rc = sprintf(ptr,"%s.totals.reject_sess "UINT64F" %llu\n",hostname, atomic64_get_na(&ts.total_rejected_sess),(unsigned long long)g_now.tv_sec); ptr += rc; + rc = sprintf(ptr,"%s.totals.reject_sess "UINT64F" %llu\n",hostname, atomic64_get_na(&ts->total_rejected_sess),(unsigned long long)g_now.tv_sec); ptr += rc; ilog(LOG_DEBUG, "min_sessions:%llu max_sessions:%llu, call_dur_per_interval:%llu.%06llu at time %llu\n", - (unsigned long long) ts.managed_sess_min, - (unsigned long long) ts.managed_sess_max, - (unsigned long long ) ts.total_calls_duration_interval.tv_sec, - (unsigned long long ) ts.total_calls_duration_interval.tv_usec, + (unsigned long long) ts->managed_sess_min, + (unsigned long long) ts->managed_sess_max, + (unsigned long long ) ts->total_calls_duration_interval.tv_sec, + (unsigned long long ) ts->total_calls_duration_interval.tv_usec, (unsigned long long ) g_now.tv_sec); rc = write(graphite_sock, data_to_send, ptr - data_to_send); @@ -196,6 +198,12 @@ error: return -1; } +static inline void copy_with_lock(struct totalstats *ts_dst, struct totalstats *ts_src, mutex_t *ts_lock) { + mutex_lock(ts_lock); + memcpy(ts_dst, ts_src, sizeof(struct totalstats)); + mutex_unlock(ts_lock); +} + void graphite_loop_run(struct callmaster* callmaster, int seconds) { int rc=0; @@ -261,11 +269,14 @@ void graphite_loop_run(struct callmaster* callmaster, int seconds) { if (graphite_sock>0 && !connectinprogress) { add_total_calls_duration_in_interval(cm, &graphite_interval_tv); - rc = send_graphite_data(); + + rc = send_graphite_data(&graphite_stats); gettimeofday(&cm->latest_graphite_interval_start, NULL); if (rc<0) { ilog(LOG_ERROR,"Sending graphite data failed."); } + + copy_with_lock(&cm->totalstats_lastinterval, &graphite_stats, &cm->totalstats_lastinterval); } }