|
|
|
@ -232,163 +232,6 @@ INLINE void thread_create_detach(void (*f)(void *), void *a, const char *name) { |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/*** ATOMIC64 ***/ |
|
|
|
|
|
|
|
typedef struct { |
|
|
|
uint64_t a; |
|
|
|
} atomic64; |
|
|
|
|
|
|
|
INLINE uint64_t atomic64_get(const atomic64 *u) { |
|
|
|
return __atomic_load_n(&u->a, __ATOMIC_SEQ_CST); |
|
|
|
} |
|
|
|
INLINE uint64_t atomic64_get_na(const atomic64 *u) { |
|
|
|
return __atomic_load_n(&u->a, __ATOMIC_RELAXED); |
|
|
|
} |
|
|
|
INLINE void atomic64_set(atomic64 *u, uint64_t a) { |
|
|
|
__atomic_store_n(&u->a, a, __ATOMIC_SEQ_CST); |
|
|
|
} |
|
|
|
INLINE gboolean atomic64_set_if(atomic64 *u, uint64_t a, uint64_t i) { |
|
|
|
return __atomic_compare_exchange_n(&u->a, &i, a, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); |
|
|
|
} |
|
|
|
INLINE void atomic64_set_na(atomic64 *u, uint64_t a) { |
|
|
|
__atomic_store_n(&u->a, a, __ATOMIC_RELAXED); |
|
|
|
} |
|
|
|
INLINE uint64_t atomic64_add(atomic64 *u, uint64_t a) { |
|
|
|
return __atomic_fetch_add(&u->a, a, __ATOMIC_SEQ_CST); |
|
|
|
} |
|
|
|
INLINE uint64_t atomic64_add_na(atomic64 *u, uint64_t a) { |
|
|
|
return __atomic_fetch_add(&u->a, a, __ATOMIC_RELAXED); |
|
|
|
} |
|
|
|
INLINE uint64_t atomic64_get_set(atomic64 *u, uint64_t a) { |
|
|
|
uint64_t old; |
|
|
|
do { |
|
|
|
old = atomic64_get(u); |
|
|
|
if (__atomic_compare_exchange_n(&u->a, &old, a, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) |
|
|
|
return old; |
|
|
|
} while (1); |
|
|
|
} |
|
|
|
INLINE uint64_t atomic64_or(atomic64 *u, uint64_t a) { |
|
|
|
return __atomic_fetch_or(&u->a, a, __ATOMIC_SEQ_CST); |
|
|
|
} |
|
|
|
INLINE uint64_t atomic64_and(atomic64 *u, uint64_t a) { |
|
|
|
return __atomic_fetch_and(&u->a, a, __ATOMIC_SEQ_CST); |
|
|
|
} |
|
|
|
|
|
|
|
INLINE uint64_t atomic64_inc(atomic64 *u) { |
|
|
|
return atomic64_add(u, 1); |
|
|
|
} |
|
|
|
INLINE uint64_t atomic64_inc_na(atomic64 *u) { |
|
|
|
return atomic64_add_na(u, 1); |
|
|
|
} |
|
|
|
INLINE uint64_t atomic64_dec(atomic64 *u) { |
|
|
|
return atomic64_add(u, -1); |
|
|
|
} |
|
|
|
INLINE void atomic64_local_copy_zero(atomic64 *dst, atomic64 *src) { |
|
|
|
atomic64_set_na(dst, atomic64_get_set(src, 0)); |
|
|
|
} |
|
|
|
#define atomic64_local_copy_zero_struct(d, s, member) \ |
|
|
|
atomic64_local_copy_zero(&((d)->member), &((s)->member)) |
|
|
|
|
|
|
|
INLINE void atomic64_min(atomic64 *min, uint64_t val) { |
|
|
|
do { |
|
|
|
uint64_t old = atomic64_get(min); |
|
|
|
if (old && old <= val) |
|
|
|
break; |
|
|
|
if (atomic64_set_if(min, val, old)) |
|
|
|
break; |
|
|
|
} while (1); |
|
|
|
} |
|
|
|
INLINE void atomic64_max(atomic64 *max, uint64_t val) { |
|
|
|
do { |
|
|
|
uint64_t old = atomic64_get(max); |
|
|
|
if (old && old >= val) |
|
|
|
break; |
|
|
|
if (atomic64_set_if(max, val, old)) |
|
|
|
break; |
|
|
|
} while (1); |
|
|
|
} |
|
|
|
|
|
|
|
INLINE void atomic64_calc_rate_from_diff(long long run_diff_us, uint64_t diff, atomic64 *rate_var) { |
|
|
|
atomic64_set(rate_var, run_diff_us ? diff * 1000000LL / run_diff_us : 0); |
|
|
|
} |
|
|
|
INLINE void atomic64_calc_rate(const atomic64 *ax_var, long long run_diff_us, |
|
|
|
atomic64 *intv_var, atomic64 *rate_var) |
|
|
|
{ |
|
|
|
uint64_t ax = atomic64_get(ax_var); |
|
|
|
uint64_t old_intv = atomic64_get(intv_var); |
|
|
|
atomic64_set(intv_var, ax); |
|
|
|
atomic64_calc_rate_from_diff(run_diff_us, ax - old_intv, rate_var); |
|
|
|
} |
|
|
|
INLINE void atomic64_calc_diff(const atomic64 *ax_var, atomic64 *intv_var, atomic64 *diff_var) { |
|
|
|
uint64_t ax = atomic64_get(ax_var); |
|
|
|
uint64_t old_intv = atomic64_get(intv_var); |
|
|
|
atomic64_set(intv_var, ax); |
|
|
|
atomic64_set(diff_var, ax - old_intv); |
|
|
|
} |
|
|
|
INLINE void atomic64_mina(atomic64 *min, atomic64 *inp) { |
|
|
|
atomic64_min(min, atomic64_get(inp)); |
|
|
|
} |
|
|
|
INLINE void atomic64_maxa(atomic64 *max, atomic64 *inp) { |
|
|
|
atomic64_max(max, atomic64_get(inp)); |
|
|
|
} |
|
|
|
INLINE double atomic64_div(const atomic64 *n, const atomic64 *d) { |
|
|
|
int64_t dd = atomic64_get(d); |
|
|
|
if (!dd) |
|
|
|
return 0.; |
|
|
|
return (double) atomic64_get(n) / (double) dd; |
|
|
|
} |
|
|
|
|
|
|
|
#define atomic_get_na(x) __atomic_load_n(x, __ATOMIC_RELAXED) |
|
|
|
#define atomic_set_na(x,y) __atomic_store_n(x, y, __ATOMIC_RELAXED) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/*** ATOMIC BITFIELD OPERATIONS ***/ |
|
|
|
|
|
|
|
/* checks if at least one of the flags is set */ |
|
|
|
INLINE bool bf_isset(const atomic64 *u, const uint64_t f) { |
|
|
|
if ((atomic64_get(u) & f)) |
|
|
|
return true; |
|
|
|
return false; |
|
|
|
} |
|
|
|
/* checks if all of the flags are set */ |
|
|
|
INLINE bool bf_areset(const atomic64 *u, const uint64_t f) { |
|
|
|
if ((atomic64_get(u) & f) == f) |
|
|
|
return true; |
|
|
|
return false; |
|
|
|
} |
|
|
|
/* returns true if at least one of the flags was set already */ |
|
|
|
INLINE bool bf_set(atomic64 *u, const uint64_t f) { |
|
|
|
return (atomic64_or(u, f) & f) ? true : false; |
|
|
|
} |
|
|
|
/* returns true if at least one of the flags was set */ |
|
|
|
INLINE bool bf_clear(atomic64 *u, const uint64_t f) { |
|
|
|
return (atomic64_and(u, ~f) & f) ? true : false; |
|
|
|
} |
|
|
|
INLINE void bf_set_clear(atomic64 *u, const uint64_t f, bool cond) { |
|
|
|
if (cond) |
|
|
|
bf_set(u, f); |
|
|
|
else |
|
|
|
bf_clear(u, f); |
|
|
|
} |
|
|
|
/* works only for single flags */ |
|
|
|
INLINE void bf_copy(atomic64 *u, const uint64_t f, |
|
|
|
const atomic64 *s, const uint64_t g) |
|
|
|
{ |
|
|
|
bf_set_clear(u, f, bf_isset(s, g)); |
|
|
|
} |
|
|
|
/* works for multiple flags */ |
|
|
|
INLINE void bf_copy_same(atomic64 *u, const atomic64 *s, const uint64_t g) { |
|
|
|
unsigned int old, set, clear; |
|
|
|
old = atomic64_get(s); |
|
|
|
set = old & g; |
|
|
|
clear = ~old & g; |
|
|
|
bf_set(u, set); |
|
|
|
bf_clear(u, clear); |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/*** STATS HELPERS ***/ |
|
|
|
|
|
|
|
#define STAT_MIN_MAX_RESET_ZERO(x, mm, loc) \ |
|
|
|
|
Richard Fuchs
2 years ago