diff --git a/rpmalloc/catomic.h b/rpmalloc/catomic.h index 47ad3ae..bcdfb3f 100644 --- a/rpmalloc/catomic.h +++ b/rpmalloc/catomic.h @@ -2626,10 +2626,10 @@ make_atomic(void *, atomic_ptr_t) #if defined(__i386__) || defined(__ppc__) || defined(__arm__) || defined(_M_ARM) || defined(__i386) || defined(_M_IX86) /* reads a value from an atomic object then cast to type */ -# define atomic_get(type, obj) (type)c89atomic_load_32((c89atomic_uint32 *)obj) +# define atomic_get(type, obj) (type)c89atomic_load_32((atomic_uint *)obj) #else /* reads a value from an atomic object then cast to type */ -# define atomic_get(type, obj) (type)c89atomic_load_64((c89atomic_uint64 *)obj) +# define atomic_get(type, obj) (type)c89atomic_load_64((atomic_ullong *)obj) #endif #if !defined(_STDATOMIC_H) @@ -2654,130 +2654,130 @@ make_atomic(void *, atomic_ptr_t) #if defined(__i386__) || defined(__ppc__) || defined(__arm__) || defined(_M_ARM) || defined(__i386) || defined(_M_IX86) /* sets an atomic_flag to true and returns the old value */ -#define atomic_is_lock_free(obj) c89atomic_is_lock_free_32((c89atomic_uint32 *)obj) +#define atomic_is_lock_free(obj) c89atomic_is_lock_free_32((atomic_uint *)obj) /* stores a value in an atomic object */ -#define atomic_store(obj, desired) c89atomic_store_32((c89atomic_uint32 *)obj, (c89atomic_uint32)desired) +#define atomic_store(obj, desired) c89atomic_store_32((atomic_uint *)obj, (c89atomic_uint32)desired) /* stores a value in an atomic object */ -#define atomic_store_explicit(obj, desired, order) c89atomic_store_explicit_32((c89atomic_uint32 *)obj, (c89atomic_uint32)desired, order) +#define atomic_store_explicit(obj, desired, order) c89atomic_store_explicit_32((atomic_uint *)obj, (c89atomic_uint32)desired, order) /* reads a value from an atomic object */ -#define atomic_load(obj) c89atomic_load_32((c89atomic_uint32 *)obj) +#define atomic_load(obj) c89atomic_load_32((atomic_uint *)obj) /* reads a value from an atomic object */ -#define atomic_load_explicit(obj, order) c89atomic_load_explicit_32((c89atomic_uint32 *)obj, order) +#define atomic_load_explicit(obj, order) c89atomic_load_explicit_32((atomic_uint *)obj, order) /* initializes an existing atomic object */ -#define atomic_init(obj, desired) c89atomic_store_32((c89atomic_uint32 *)obj, (c89atomic_uint32)desired) +#define atomic_init(obj, desired) c89atomic_store_32((atomic_uint *)obj, (c89atomic_uint32)desired) /* atomic addition */ -#define atomic_fetch_add(obj, arg) c89atomic_fetch_add_32((c89atomic_uint32 *)obj, (c89atomic_uint32)arg) +#define atomic_fetch_add(obj, arg) c89atomic_fetch_add_32((atomic_uint *)obj, (c89atomic_uint32)arg) /* atomic addition */ -#define atomic_fetch_add_explicit(obj, arg, order) c89atomic_fetch_add_explicit_32((c89atomic_uint32 *)obj, (c89atomic_uint32)arg, order) +#define atomic_fetch_add_explicit(obj, arg, order) c89atomic_fetch_add_explicit_32((atomic_uint *)obj, (c89atomic_uint32)arg, order) /* atomic subtraction */ -#define atomic_fetch_sub(obj, arg) c89atomic_fetch_sub_32((c89atomic_uint32 *)obj, (c89atomic_uint32)arg) +#define atomic_fetch_sub(obj, arg) c89atomic_fetch_sub_32((atomic_uint *)obj, (c89atomic_uint32)arg) /* atomic subtraction */ -#define atomic_fetch_sub_explicit(obj, arg, order) c89atomic_fetch_sub_explicit_32((c89atomic_uint32 *)obj, (c89atomic_uint32)arg, order) +#define atomic_fetch_sub_explicit(obj, arg, order) c89atomic_fetch_sub_explicit_32((atomic_uint *)obj, (c89atomic_uint32)arg, order) /* atomic bitwise OR */ -#define atomic_fetch_or(obj, arg) c89atomic_fetch_or_32((c89atomic_uint32 *)obj, (c89atomic_uint32)arg) +#define atomic_fetch_or(obj, arg) c89atomic_fetch_or_32((atomic_uint *)obj, (c89atomic_uint32)arg) /* atomic bitwise OR */ -#define atomic_fetch_or_explicit(obj, arg, order) c89atomic_fetch_or_explicit_32((c89atomic_uint32 *)obj, (c89atomic_uint32)arg, order) +#define atomic_fetch_or_explicit(obj, arg, order) c89atomic_fetch_or_explicit_32((atomic_uint *)obj, (c89atomic_uint32)arg, order) /* atomic bitwise exclusive OR */ -#define atomic_fetch_xor(obj, arg) c89atomic_fetch_xor_32((c89atomic_uint32 *)obj, (c89atomic_uint32)arg) +#define atomic_fetch_xor(obj, arg) c89atomic_fetch_xor_32((atomic_uint *)obj, (c89atomic_uint32)arg) /* atomic bitwise exclusive OR */ -#define atomic_fetch_xor_explicit(obj, arg, order) c89atomic_fetch_xor_explicit_32((c89atomic_uint32 *)obj, (c89atomic_uint32)arg, order) +#define atomic_fetch_xor_explicit(obj, arg, order) c89atomic_fetch_xor_explicit_32((atomic_uint *)obj, (c89atomic_uint32)arg, order) /* atomic bitwise AND */ #define atomic_fetch_and(obj, arg) \ - c89atomic_fetch_and_32((c89atomic_uint32 *)obj, (c89atomic_uint32)arg) + c89atomic_fetch_and_32((atomic_uint *)obj, (c89atomic_uint32)arg) /* atomic bitwise AND */ #define atomic_fetch_and_explicit(obj, arg, order) \ - c89atomic_fetch_and_explicit_32((c89atomic_uint32 *)obj, (c89atomic_uint32)arg, order) + c89atomic_fetch_and_explicit_32((atomic_uint *)obj, (c89atomic_uint32)arg, order) /* swaps a value with the value of an atomic object */ #define atomic_exchange(obj, desired) \ - c89atomic_exchange_32((c89atomic_uint32 *)obj, (c89atomic_uint32)desired) + c89atomic_exchange_32((atomic_uint *)obj, (c89atomic_uint32)desired) /* swaps a value with the value of an atomic object */ #define atomic_exchange_explicit(obj, desired, order) \ - c89atomic_exchange_explicit_32((c89atomic_uint32 *)obj, (c89atomic_uint32)desired, order) + c89atomic_exchange_explicit_32((atomic_uint *)obj, (c89atomic_uint32)desired, order) /* swaps a value with an atomic object if the old value is what is expected, otherwise reads the old value */ -#define atomic_compare_exchange_weak(obj, expected, desired) atomic_cas_32((c89atomic_uint32 *)obj, (c89atomic_uint32)expected, desired) +#define atomic_compare_exchange_weak(obj, expected, desired) atomic_cas_32((atomic_uint *)obj, (c89atomic_uint32)expected, desired) /* swaps a value with an atomic object if the old value is what is expected, otherwise reads the old value */ #define atomic_compare_exchange_weak_explicit(obj, expected, desired, succ, fail) \ - c89atomic_compare_exchange_weak_explicit_32((c89atomic_uint32 *)obj, (c89atomic_uint32)expected, (c89atomic_uint32)desired, succ, fail) + c89atomic_compare_exchange_weak_explicit_32((atomic_uint *)obj, (c89atomic_uint32)expected, (c89atomic_uint32)desired, succ, fail) /* swaps a value with an atomic object if the old value is what is expected, otherwise reads the old value */ -#define atomic_compare_exchange_strong(obj, expected, desired) atomic_cas_32((c89atomic_uint32 *)obj, (c89atomic_uint32)expected, desired) +#define atomic_compare_exchange_strong(obj, expected, desired) atomic_cas_32((atomic_uint *)obj, (c89atomic_uint32)expected, desired) /* swaps a value with an atomic object if the old value is what is expected, otherwise reads the old value */ #define atomic_compare_exchange_strong_explicit(obj, expected, desired, succ, fail) \ - c89atomic_compare_exchange_strong_explicit_32((c89atomic_uint32 *)obj, (c89atomic_uint32)expected, (c89atomic_uint32)desired, succ, fail) + c89atomic_compare_exchange_strong_explicit_32((atomic_uint *)obj, (c89atomic_uint32)expected, (c89atomic_uint32)desired, succ, fail) #else /* indicates whether the atomic object is lock-free */ -#define atomic_is_lock_free(obj) c89atomic_is_lock_free_64((c89atomic_uint64 *)obj) +#define atomic_is_lock_free(obj) c89atomic_is_lock_free_64((atomic_ullong *)obj) /* stores a value in an atomic object */ -#define atomic_store(obj, desired) c89atomic_store_64((c89atomic_uint64 *)obj, (c89atomic_uint64)desired) +#define atomic_store(obj, desired) c89atomic_store_64((atomic_ullong *)obj, (c89atomic_uint64)desired) /* stores a value in an atomic object */ -#define atomic_store_explicit(obj, desired, order) c89atomic_store_explicit_64((c89atomic_uint64 *)obj, (c89atomic_uint64)desired, order) +#define atomic_store_explicit(obj, desired, order) c89atomic_store_explicit_64((atomic_ullong *)obj, (c89atomic_uint64)desired, order) /* reads a value from an atomic object */ -#define atomic_load(obj) c89atomic_load_64((c89atomic_uint64 *)obj) +#define atomic_load(obj) c89atomic_load_64((atomic_ullong *)obj) /* reads a value from an atomic object */ -#define atomic_load_explicit(obj, order) c89atomic_load_explicit_64((c89atomic_uint64 *)obj, order) +#define atomic_load_explicit(obj, order) c89atomic_load_explicit_64((atomic_ullong *)obj, order) /* initializes an existing atomic object */ -#define atomic_init(obj, desired) c89atomic_store_64((c89atomic_uint64 *)obj, (c89atomic_uint64)desired) +#define atomic_init(obj, desired) c89atomic_store_64((atomic_ullong *)obj, (c89atomic_uint64)desired) /* atomic addition */ -#define atomic_fetch_add(obj, arg) c89atomic_fetch_add_64((c89atomic_uint64 *)obj, (c89atomic_uint64)arg) +#define atomic_fetch_add(obj, arg) c89atomic_fetch_add_64((atomic_ullong *)obj, (c89atomic_uint64)arg) /* atomic addition */ -#define atomic_fetch_add_explicit(obj, arg, order) c89atomic_fetch_add_explicit_64((c89atomic_uint64 *)obj, (c89atomic_uint64)arg, order) +#define atomic_fetch_add_explicit(obj, arg, order) c89atomic_fetch_add_explicit_64((atomic_ullong *)obj, (c89atomic_uint64)arg, order) /* atomic subtraction */ -#define atomic_fetch_sub(obj, arg) c89atomic_fetch_sub_64((c89atomic_uint64 *)obj, (c89atomic_uint64)arg) +#define atomic_fetch_sub(obj, arg) c89atomic_fetch_sub_64((atomic_ullong *)obj, (c89atomic_uint64)arg) /* atomic subtraction */ #define atomic_fetch_sub_explicit(obj, arg, order) c89atomic_fetch_sub_explicit_64(obj, (c89atomic_uint64)arg, order) /* atomic bitwise OR */ -#define atomic_fetch_or(obj, arg) c89atomic_fetch_or_64((c89atomic_uint64 *)obj, (c89atomic_uint64 *)arg) +#define atomic_fetch_or(obj, arg) c89atomic_fetch_or_64((atomic_ullong *)obj, (atomic_ullong *)arg) /* atomic bitwise OR */ -#define atomic_fetch_or_explicit(obj, arg, order) c89atomic_fetch_or_explicit_64((c89atomic_uint64 *)obj, (c89atomic_uint64)arg, order) +#define atomic_fetch_or_explicit(obj, arg, order) c89atomic_fetch_or_explicit_64((atomic_ullong *)obj, (c89atomic_uint64)arg, order) /* atomic bitwise exclusive OR */ #define atomic_fetch_xor(obj, arg) \ - c89atomic_fetch_xor_64((c89atomic_uint64 *)obj, (c89atomic_uint64)arg) + c89atomic_fetch_xor_64((atomic_ullong *)obj, (c89atomic_uint64)arg) /* atomic bitwise exclusive OR */ #define atomic_fetch_xor_explicit(obj, arg, order) \ - c89atomic_fetch_xor_explicit_64((c89atomic_uint64 *)obj, (c89atomic_uint64)arg, order) + c89atomic_fetch_xor_explicit_64((atomic_ullong *)obj, (c89atomic_uint64)arg, order) /* atomic bitwise AND */ #define atomic_fetch_and(obj, arg) \ - c89atomic_fetch_and_64((c89atomic_uint64 *)obj, (c89atomic_uint64)arg) + c89atomic_fetch_and_64((atomic_ullong *)obj, (c89atomic_uint64)arg) /* atomic bitwise AND */ #define atomic_fetch_and_explicit(obj, arg, order) \ - c89atomic_fetch_and_explicit_64((c89atomic_uint64 *)obj, (c89atomic_uint64)arg, order) + c89atomic_fetch_and_explicit_64((atomic_ullong *)obj, (c89atomic_uint64)arg, order) /* swaps a value with the value of an atomic object */ #define atomic_exchange(obj, desired) \ - c89atomic_exchange_64((c89atomic_uint64 *)obj, (c89atomic_uint64)desired) + c89atomic_exchange_64((atomic_ullong *)obj, (c89atomic_uint64)desired) /* swaps a value with the value of an atomic object */ #define atomic_exchange_explicit(obj, desired, order) \ - c89atomic_exchange_explicit_64((c89atomic_uint64 *)obj, (c89atomic_uint64)(c89atomic_uint64)desired, order) + c89atomic_exchange_explicit_64((atomic_ullong *)obj, (c89atomic_uint64)(c89atomic_uint64)desired, order) /* swaps a value with an atomic object if the old value is what is expected, otherwise reads the old value */ -#define atomic_compare_exchange_weak(obj, expected, desired) atomic_cas((c89atomic_uint64 *)obj, expected, desired) +#define atomic_compare_exchange_weak(obj, expected, desired) atomic_cas((atomic_ullong *)obj, expected, desired) /* swaps a value with an atomic object if the old value is what is expected, otherwise reads the old value */ #define atomic_compare_exchange_weak_explicit(obj, expected, desired, succ, fail) \ - c89atomic_compare_exchange_weak_explicit_64((c89atomic_uint64 *)obj, (c89atomic_uint64)expected, (c89atomic_uint64)desired, succ, fail) + c89atomic_compare_exchange_weak_explicit_64((atomic_ullong *)obj, (c89atomic_uint64)expected, (c89atomic_uint64)desired, succ, fail) /* swaps a value with an atomic object if the old value is what is expected, otherwise reads the old value */ -#define atomic_compare_exchange_strong(obj, expected, desired) atomic_cas((c89atomic_uint64 *)obj, expected, desired) +#define atomic_compare_exchange_strong(obj, expected, desired) atomic_cas((atomic_ullong *)obj, expected, desired) /* swaps a value with an atomic object if the old value is what is expected, otherwise reads the old value */ #define atomic_compare_exchange_strong_explicit(obj, expected, desired, succ, fail) \ - c89atomic_compare_exchange_strong_explicit_64((c89atomic_uint64 *)obj, (c89atomic_uint64)expected, (c89atomic_uint64)desired, succ, fail) + c89atomic_compare_exchange_strong_explicit_64((atomic_ullong *)obj, (c89atomic_uint64)expected, (c89atomic_uint64)desired, succ, fail) #endif #endif diff --git a/rpmalloc/rpmalloc.c b/rpmalloc/rpmalloc.c index 6b126fc..518ae11 100644 --- a/rpmalloc/rpmalloc.c +++ b/rpmalloc/rpmalloc.c @@ -261,20 +261,49 @@ static int _rpmalloc_shuting_down = 0; make_atomic(unsigned int, atomic32_t) make_atomic(unsigned long long, atomic64_t) -static FORCEINLINE int32_t atomic_load32(atomic32_t *src) { return c89atomic_load_explicit_32(src, memory_order_relaxed); } -static FORCEINLINE void atomic_store32(atomic32_t *dst, int32_t val) { c89atomic_store_explicit_32(dst, val, memory_order_relaxed); } -static FORCEINLINE int32_t atomic_incr32(atomic32_t *val) { return c89atomic_fetch_add_explicit_32(val, 1, memory_order_relaxed) + 1; } -static FORCEINLINE int32_t atomic_decr32(atomic32_t *val) { return c89atomic_fetch_add_explicit_32(val, -1, memory_order_relaxed) - 1; } -static FORCEINLINE int32_t atomic_add32(atomic32_t *val, int32_t add) { return c89atomic_fetch_add_explicit_32(val, add, memory_order_relaxed) + add; } -static FORCEINLINE int atomic_cas32_acquire(atomic32_t *dst, int32_t val, int32_t ref) { return c89atomic_compare_exchange_weak_explicit_32(dst, &ref, val, memory_order_acquire, memory_order_relaxed); } -static FORCEINLINE void atomic_store32_release(atomic32_t *dst, int32_t val) { c89atomic_store_explicit_32(dst, val, memory_order_release); } -static FORCEINLINE int64_t atomic_load64(atomic64_t *val) { return c89atomic_load_explicit_64((volatile c89atomic_uint64 *)val, memory_order_relaxed); } -static FORCEINLINE int64_t atomic_add64(atomic64_t *val, int64_t add) { return c89atomic_fetch_add_explicit_64((volatile c89atomic_uint64 *)val, add, memory_order_relaxed) + add; } -static FORCEINLINE void *atomic_load_ptr(atomic_ptr_t *src) { return (void *)c89atomic_load_explicit_64((volatile c89atomic_uint64 *)src, memory_order_relaxed); } -static FORCEINLINE void atomic_store_ptr(atomic_ptr_t *dst, void *val) { c89atomic_store_explicit_64((volatile c89atomic_uint64 *)dst, (c89atomic_uint64)val, memory_order_relaxed); } -static FORCEINLINE void atomic_store_ptr_release(atomic_ptr_t *dst, void *val) { c89atomic_store_explicit_64((volatile c89atomic_uint64 *)dst, (c89atomic_uint64)val, memory_order_release); } -static FORCEINLINE void *atomic_exchange_ptr_acquire(atomic_ptr_t *dst, void *val) { return (void *)c89atomic_exchange_explicit_64((volatile c89atomic_uint64 *)dst, (c89atomic_uint64)val, memory_order_acquire); } -static FORCEINLINE int atomic_cas_ptr(atomic_ptr_t *dst, void *val, void *ref) { return (int)atomic_swap(dst, &ref, (c89atomic_uint64)val); } +static FORCEINLINE int32_t atomic_load32(atomic32_t * src) { + return c89atomic_load_explicit_32(src, memory_order_relaxed); +} +static FORCEINLINE void atomic_store32(atomic32_t *dst, int32_t val) { + c89atomic_store_explicit_32(dst, val, memory_order_relaxed); +} +static FORCEINLINE int32_t atomic_incr32(atomic32_t *val) { + return c89atomic_fetch_add_explicit_32(val, 1, memory_order_relaxed) + 1; +} +static FORCEINLINE int32_t atomic_decr32(atomic32_t *val) { + return c89atomic_fetch_add_explicit_32(val, -1, memory_order_relaxed) - 1; +} +static FORCEINLINE int32_t atomic_add32(atomic32_t *val, int32_t add) { + return c89atomic_fetch_add_explicit_32(val, add, memory_order_relaxed) + add; +} +static FORCEINLINE int atomic_cas32_acquire(atomic32_t *dst, int32_t val, int32_t ref) { + return c89atomic_compare_exchange_weak_explicit_32(dst, &ref, val, memory_order_acquire, memory_order_relaxed); +} +static FORCEINLINE void atomic_store32_release(atomic32_t *dst, int32_t val) { + c89atomic_store_explicit_32(dst, val, memory_order_release); +} +static FORCEINLINE int64_t atomic_load64(atomic64_t *val) { + return c89atomic_load_explicit_64((volatile c89atomic_uint64 *)val, memory_order_relaxed); +} +static FORCEINLINE int64_t atomic_add64(atomic64_t *val, int64_t add) { + return c89atomic_fetch_add_explicit_64((volatile c89atomic_uint64 *)val, add, memory_order_relaxed) + add; +} + +static FORCEINLINE void *atomic_load_ptr(atomic_ptr_t *src) { + return (void *)atomic_load_explicit(src, memory_order_relaxed); +} +static FORCEINLINE void atomic_store_ptr(atomic_ptr_t *dst, void *val) { + atomic_store_explicit(dst, val, memory_order_relaxed); +} +static FORCEINLINE void atomic_store_ptr_release(atomic_ptr_t *dst, void *val) { + atomic_store_explicit(dst, val, memory_order_release); +} +static FORCEINLINE void *atomic_exchange_ptr_acquire(atomic_ptr_t *dst, void *val) { + return (void *)atomic_exchange_explicit(dst, val, memory_order_acquire); +} +static FORCEINLINE int atomic_cas_ptr(atomic_ptr_t *dst, void *val, void *ref) { + return (int)atomic_swap(dst, &ref, val); +} #if defined(__TINYC__) || !defined(_WIN32) int rpmalloc_tls_create(tls_t *key, tls_dtor_t dtor) {