@@ -540,10 +540,10 @@ int main(int argc, char *argv[])
}
/* init counters */
- odp_atomic_init_u64(&counters.seq);
- odp_atomic_init_u64(&counters.ip);
- odp_atomic_init_u64(&counters.udp);
- odp_atomic_init_u64(&counters.icmp);
+ odp_atomic_init_u64(&counters.seq, 0);
+ odp_atomic_init_u64(&counters.ip, 0);
+ odp_atomic_init_u64(&counters.udp, 0);
+ odp_atomic_init_u64(&counters.icmp, 0);
/* Reserve memory for args from shared mem */
shm = odp_shm_reserve("shm_args", sizeof(args_t),
@@ -18,11 +18,12 @@
extern "C" {
#endif
-
-#include <odp_std_types.h>
+#include <stdint.h>
+#include <odp_align.h>
/** @addtogroup odp_synchronizers
- * Atomic operations.
+ * Atomic types and relaxed operations. These operations cannot be used for
+ * synchronization.
* @{
*/
@@ -30,56 +31,67 @@ extern "C" {
/**
* Atomic unsigned integer 64 bits
*/
-typedef volatile uint64_t odp_atomic_u64_t;
+typedef struct {
+ uint64_t v; /**< Actual storage for the atomic variable */
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ /* Some architectures do not support lock-free operations on 64-bit
+ * data types. We use a spin lock to ensure atomicity. */
+ char lock;
+#endif
+} odp_atomic_u64_t
+ODP_ALIGNED(sizeof(uint64_t)); /* Enforce alignement! */
+
/**
* Atomic unsigned integer 32 bits
*/
-typedef volatile uint32_t odp_atomic_u32_t;
+typedef struct {
+ uint32_t v; /**< Actual storage for the atomic variable */
+} odp_atomic_u32_t
+ODP_ALIGNED(sizeof(uint32_t)); /* Enforce alignement! */
/**
* Initialize atomic uint32
+ * @note Relaxed memory order, cannot be used for synchronization
*
* @param ptr An atomic variable
- *
- * @note The operation is not synchronized with other threads
+ * @param val Value to initialize the variable with
*/
-static inline void odp_atomic_init_u32(odp_atomic_u32_t *ptr)
+static inline void odp_atomic_init_u32(odp_atomic_u32_t *ptr, uint32_t val)
{
- *ptr = 0;
+ __atomic_store_n(&ptr->v, val, __ATOMIC_RELAXED);
}
/**
* Load value of atomic uint32
+ * @note Relaxed memory order, cannot be used for synchronization
*
* @param ptr An atomic variable
*
- * @return atomic uint32 value
- *
- * @note The operation is not synchronized with other threads
+ * @return Value of the variable
*/
static inline uint32_t odp_atomic_load_u32(odp_atomic_u32_t *ptr)
{
- return *ptr;
+ return __atomic_load_n(&ptr->v, __ATOMIC_RELAXED);
}
/**
* Store value to atomic uint32
+ * @note Relaxed memory order, cannot be used for synchronization
*
* @param ptr An atomic variable
* @param new_value Store new_value to a variable
- *
- * @note The operation is not synchronized with other threads
*/
static inline void odp_atomic_store_u32(odp_atomic_u32_t *ptr,
uint32_t new_value)
{
- *ptr = new_value;
+ __atomic_store_n(&ptr->v, new_value, __ATOMIC_RELAXED);
}
/**
* Fetch and add atomic uint32
+ * @note Relaxed memory order, cannot be used for synchronization
*
* @param ptr An atomic variable
* @param value A value to be added to the variable
@@ -89,11 +101,25 @@ static inline void odp_atomic_store_u32(odp_atomic_u32_t *ptr,
static inline uint32_t odp_atomic_fetch_add_u32(odp_atomic_u32_t *ptr,
uint32_t value)
{
- return __sync_fetch_and_add(ptr, value);
+ return __atomic_fetch_add(&ptr->v, value, __ATOMIC_RELAXED);
+}
+
+/**
+ * Add atomic uint32
+ * @note Relaxed memory order, cannot be used for synchronization
+ *
+ * @param ptr An atomic variable
+ * @param value A value to be added to the variable
+ */
+static inline void odp_atomic_add_u32(odp_atomic_u32_t *ptr,
+ uint32_t value)
+{
+ (void)__atomic_fetch_add(&ptr->v, value, __ATOMIC_RELAXED);
}
/**
* Fetch and subtract uint32
+ * @note Relaxed memory order, cannot be used for synchronization
*
* @param ptr An atomic variable
* @param value A value to be sub to the variable
@@ -103,51 +129,59 @@ static inline uint32_t odp_atomic_fetch_add_u32(odp_atomic_u32_t *ptr,
static inline uint32_t odp_atomic_fetch_sub_u32(odp_atomic_u32_t *ptr,
uint32_t value)
{
- return __sync_fetch_and_sub(ptr, value);
+ return __atomic_fetch_sub(&ptr->v, value, __ATOMIC_RELAXED);
+}
+
+/**
+ * Subtract uint32
+ * @note Relaxed memory order, cannot be used for synchronization
+ *
+ * @param ptr An atomic variable
+ * @param value A value to be subtract from the variable
+ */
+static inline void odp_atomic_sub_u32(odp_atomic_u32_t *ptr,
+ uint32_t value)
+{
+ (void)__atomic_fetch_sub(&ptr->v, value, __ATOMIC_RELAXED);
}
/**
* Fetch and increment atomic uint32 by 1
+ * @note Relaxed memory order, cannot be used for synchronization
*
* @param ptr An atomic variable
*
* @return Value of the variable before the operation
*/
-#if defined __OCTEON__
static inline uint32_t odp_atomic_fetch_inc_u32(odp_atomic_u32_t *ptr)
{
+#if defined __OCTEON__
uint32_t ret;
-
__asm__ __volatile__ ("syncws");
__asm__ __volatile__ ("lai %0,(%2)" : "=r" (ret), "+m" (ptr) :
"r" (ptr));
-
return ret;
-}
-
#else
-
-static inline uint32_t odp_atomic_fetch_inc_u32(odp_atomic_u32_t *ptr)
-{
- return odp_atomic_fetch_add_u32(ptr, 1);
-}
-
+ return __atomic_fetch_add(&ptr->v, 1, __ATOMIC_RELAXED);
#endif
+}
/**
* Increment atomic uint32 by 1
+ * @note Relaxed memory order, cannot be used for synchronization
*
* @param ptr An atomic variable
*
*/
static inline void odp_atomic_inc_u32(odp_atomic_u32_t *ptr)
{
- odp_atomic_fetch_add_u32(ptr, 1);
+ (void)__atomic_fetch_add(&ptr->v, 1, __ATOMIC_RELAXED);
}
/**
* Fetch and decrement uint32 by 1
+ * @note Relaxed memory order, cannot be used for synchronization
*
* @param ptr An atomic variable
*
@@ -155,62 +189,77 @@ static inline void odp_atomic_inc_u32(odp_atomic_u32_t *ptr)
*/
static inline uint32_t odp_atomic_fetch_dec_u32(odp_atomic_u32_t *ptr)
{
- return odp_atomic_fetch_sub_u32(ptr, 1);
+ return __atomic_fetch_sub(&ptr->v, 1, __ATOMIC_RELAXED);
}
/**
* Decrement atomic uint32 by 1
+ * @note Relaxed memory order, cannot be used for synchronization
*
* @param ptr An atomic variable
*
*/
static inline void odp_atomic_dec_u32(odp_atomic_u32_t *ptr)
{
- odp_atomic_fetch_sub_u32(ptr, 1);
+ (void)__atomic_fetch_sub(&ptr->v, 1, __ATOMIC_RELAXED);
}
/**
- * Atomic compare and set for 32bit
+ * Initialize atomic uint64
+ * @note Relaxed memory order, cannot be used for synchronization
*
- * @param dst destination location into which the value will be written.
- * @param exp expected value.
- * @param src new value.
- * @return Non-zero on success; 0 on failure.
+ * @param ptr An atomic variable
+ * @param val Value to initialize the variable with
*/
-static inline int
-odp_atomic_cmpset_u32(odp_atomic_u32_t *dst, uint32_t exp, uint32_t src)
+static inline void odp_atomic_init_u64(odp_atomic_u64_t *ptr, uint64_t val)
{
- return __sync_bool_compare_and_swap(dst, exp, src);
+ ptr->v = val;
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ __atomic_clear(&ptr->lock, __ATOMIC_RELAXED);
+#endif
}
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
/**
- * Initialize atomic uint64
- *
- * @param ptr An atomic variable
- *
- * @note The operation is not synchronized with other threads
+ * @internal
+ * Helper macro for lock-based atomic operations on 64-bit integers
+ * @param ptr Pointer to the 64-bit atomic variable
+ * @param expr Expression used update the variable.
+ * @return The old value of the variable.
*/
-static inline void odp_atomic_init_u64(odp_atomic_u64_t *ptr)
-{
- *ptr = 0;
-}
+#define ATOMIC_OP(ptr, expr) \
+({ \
+ uint64_t old_val; \
+ /* Loop while lock is already taken, stop when lock becomes clear */ \
+ while (__atomic_test_and_set(&(ptr)->lock, __ATOMIC_ACQUIRE)) \
+ (void)0; \
+ old_val = (ptr)->v; \
+ (expr); /* Perform whatever update is desired */ \
+ __atomic_clear(&(ptr)->lock, __ATOMIC_RELEASE); \
+ old_val; /* Return old value */ \
+})
+#endif
/**
* Load value of atomic uint64
+ * @note Relaxed memory order, cannot be used for synchronization
*
* @param ptr An atomic variable
*
* @return atomic uint64 value
- *
- * @note The operation is not synchronized with other threads
*/
static inline uint64_t odp_atomic_load_u64(odp_atomic_u64_t *ptr)
{
- return *ptr;
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ return ATOMIC_OP(ptr, (void)0);
+#else
+ return __atomic_load_n(&ptr->v, __ATOMIC_RELAXED);
+#endif
}
/**
* Store value to atomic uint64
+ * @note Relaxed memory order, cannot be used for synchronization
*
* @param ptr An atomic variable
* @param new_value Store new_value to a variable
@@ -220,23 +269,16 @@ static inline uint64_t odp_atomic_load_u64(odp_atomic_u64_t *ptr)
static inline void odp_atomic_store_u64(odp_atomic_u64_t *ptr,
uint64_t new_value)
{
- *ptr = new_value;
-}
-
-/**
- * Add atomic uint64
- *
- * @param ptr An atomic variable
- * @param value A value to be added to the variable
- *
- */
-static inline void odp_atomic_add_u64(odp_atomic_u64_t *ptr, uint64_t value)
-{
- __sync_fetch_and_add(ptr, value);
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ (void)ATOMIC_OP(ptr, ptr->v = new_value);
+#else
+ __atomic_store_n(&ptr->v, new_value, __ATOMIC_RELAXED);
+#endif
}
/**
* Fetch and add atomic uint64
+ * @note Relaxed memory order, cannot be used for synchronization
*
* @param ptr An atomic variable
* @param value A value to be added to the variable
@@ -244,56 +286,72 @@ static inline void odp_atomic_add_u64(odp_atomic_u64_t *ptr, uint64_t value)
* @return Value of the variable before the operation
*/
-#if defined __powerpc__ && !defined __powerpc64__
static inline uint64_t odp_atomic_fetch_add_u64(odp_atomic_u64_t *ptr,
uint64_t value)
{
- return __sync_fetch_and_add((odp_atomic_u32_t *)ptr,
- (uint32_t)value);
-}
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ return ATOMIC_OP(ptr, ptr->v += value);
#else
-static inline uint64_t odp_atomic_fetch_add_u64(odp_atomic_u64_t *ptr,
- uint64_t value)
-{
- return __sync_fetch_and_add(ptr, value);
-}
+ return __atomic_fetch_add(&ptr->v, value, __ATOMIC_RELAXED);
#endif
+}
+
/**
- * Subtract atomic uint64
+ * Add atomic uint64
+ * @note Relaxed memory order, cannot be used for synchronization
*
* @param ptr An atomic variable
- * @param value A value to be subtracted from the variable
+ * @param value A value to be added to the variable
*
*/
-static inline void odp_atomic_sub_u64(odp_atomic_u64_t *ptr, uint64_t value)
+static inline void odp_atomic_add_u64(odp_atomic_u64_t *ptr, uint64_t value)
{
- __sync_fetch_and_sub(ptr, value);
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ (void)ATOMIC_OP(ptr, ptr->v += value);
+#else
+ (void)__atomic_fetch_add(&ptr->v, value, __ATOMIC_RELAXED);
+#endif
}
/**
* Fetch and subtract atomic uint64
+ * @note Relaxed memory order, cannot be used for synchronization
*
* @param ptr An atomic variable
* @param value A value to be subtracted from the variable
*
* @return Value of the variable before the operation
*/
-#if defined __powerpc__ && !defined __powerpc64__
static inline uint64_t odp_atomic_fetch_sub_u64(odp_atomic_u64_t *ptr,
uint64_t value)
{
- return __sync_fetch_and_sub((odp_atomic_u32_t *)ptr,
- (uint32_t)value);
-}
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ return ATOMIC_OP(ptr, ptr->v -= value);
#else
-static inline uint64_t odp_atomic_fetch_sub_u64(odp_atomic_u64_t *ptr,
- uint64_t value)
-{
- return __sync_fetch_and_sub(ptr, value);
+ return __atomic_fetch_sub(&ptr->v, value, __ATOMIC_RELAXED);
+#endif
}
+
+/**
+ * Subtract atomic uint64
+ * @note Relaxed memory order, cannot be used for synchronization
+ *
+ * @param ptr An atomic variable
+ * @param value A value to be subtracted from the variable
+ *
+ */
+static inline void odp_atomic_sub_u64(odp_atomic_u64_t *ptr, uint64_t value)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ (void)ATOMIC_OP(ptr, ptr->v -= value);
+#else
+ (void)__atomic_fetch_sub(&ptr->v, value, __ATOMIC_RELAXED);
#endif
+}
+
/**
* Fetch and increment atomic uint64 by 1
+ * @note Relaxed memory order, cannot be used for synchronization
*
* @param ptr An atomic variable
*
@@ -301,22 +359,32 @@ static inline uint64_t odp_atomic_fetch_sub_u64(odp_atomic_u64_t *ptr,
*/
static inline uint64_t odp_atomic_fetch_inc_u64(odp_atomic_u64_t *ptr)
{
- return odp_atomic_fetch_add_u64(ptr, 1);
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ return ATOMIC_OP(ptr, ptr->v++);
+#else
+ return __atomic_fetch_add(&ptr->v, 1, __ATOMIC_RELAXED);
+#endif
}
/**
* Increment atomic uint64 by 1
+ * @note Relaxed memory order, cannot be used for synchronization
*
* @param ptr An atomic variable
*
*/
static inline void odp_atomic_inc_u64(odp_atomic_u64_t *ptr)
{
- odp_atomic_fetch_add_u64(ptr, 1);
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ (void)ATOMIC_OP(ptr, ptr->v++);
+#else
+ (void)__atomic_fetch_add(&ptr->v, 1, __ATOMIC_RELAXED);
+#endif
}
/**
* Fetch and decrement atomic uint64 by 1
+ * @note Relaxed memory order, cannot be used for synchronization
*
* @param ptr An atomic variable
*
@@ -324,32 +392,27 @@ static inline void odp_atomic_inc_u64(odp_atomic_u64_t *ptr)
*/
static inline uint64_t odp_atomic_fetch_dec_u64(odp_atomic_u64_t *ptr)
{
- return odp_atomic_fetch_sub_u64(ptr, 1);
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ return ATOMIC_OP(ptr, ptr->v--);
+#else
+ return __atomic_fetch_sub(&ptr->v, 1, __ATOMIC_RELAXED);
+#endif
}
/**
* Decrement atomic uint64 by 1
+ * @note Relaxed memory order, cannot be used for synchronization
*
* @param ptr An atomic variable
*
*/
static inline void odp_atomic_dec_u64(odp_atomic_u64_t *ptr)
{
- odp_atomic_fetch_sub_u64(ptr, 1);
-}
-
-/**
- * Atomic compare and set for 64bit
- *
- * @param dst destination location into which the value will be written.
- * @param exp expected value.
- * @param src new value.
- * @return Non-zero on success; 0 on failure.
- */
-static inline int
-odp_atomic_cmpset_u64(odp_atomic_u64_t *dst, uint64_t exp, uint64_t src)
-{
- return __sync_bool_compare_and_swap(dst, exp, src);
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ (void)ATOMIC_OP(ptr, ptr->v--);
+#else
+ (void)__atomic_fetch_sub(&ptr->v, 1, __ATOMIC_RELAXED);
+#endif
}
/**
@@ -10,9 +10,8 @@
void odp_barrier_init(odp_barrier_t *barrier, int count)
{
- barrier->count = count;
- barrier->bar = 0;
- odp_sync_stores();
+ barrier->count = (uint32_t)count;
+ odp_atomic_init_u32(&barrier->bar, 0);
}
/*
@@ -33,14 +32,16 @@ void odp_barrier_wait(odp_barrier_t *barrier)
uint32_t count;
int wasless;
- wasless = barrier->bar < barrier->count;
__atomic_thread_fence(__ATOMIC_SEQ_CST);
count = odp_atomic_fetch_inc_u32(&barrier->bar);
+ wasless = count < barrier->count;
if (count == 2*barrier->count-1) {
- barrier->bar = 0;
+ /* Wrap around *atomically* */
+ odp_atomic_sub_u32(&barrier->bar, 2 * barrier->count);
} else {
- while ((barrier->bar < barrier->count) == wasless)
+ while ((odp_atomic_load_u32(&barrier->bar) < barrier->count)
+ == wasless)
odp_spin();
}
@@ -12,9 +12,8 @@
void odp_ticketlock_init(odp_ticketlock_t *ticketlock)
{
- ticketlock->next_ticket = 0;
+ odp_atomic_init_u32(&ticketlock->next_ticket, 0);
ticketlock->cur_ticket = 0;
- odp_sync_stores();
}
@@ -152,8 +152,8 @@ void test_atomic_basic(void)
void test_atomic_init(void)
{
- odp_atomic_init_u32(&a32u);
- odp_atomic_init_u64(&a64u);
+ odp_atomic_init_u32(&a32u, 0);
+ odp_atomic_init_u64(&a64u, 0);
}
void test_atomic_store(void)
Signed-off-by: Ola Liljedahl <ola.liljedahl@linaro.org> --- (Tried to break up this patch but "git add -p" couldn't separate the different changes from each other and my git-fu isn't strong enough yet). odp_atomic.h: Use struct for 32- and 64-bit atomic types, this prevents non-atomic access to atomic variables. Emulated 64-bit atomic support for functionally challenged 32-bit architectures (e.g. PPC32). Relaxed memory model for atomic operations, this enhances performance on weakly ordered architectures (e.g. most RISC). Use GCC __atomic builtins (previously __sync builtins). Added missing atomic_add_u32, atomic_sub_u32 and atomic_sub_64 operations. Removed 32- and 64-bit cmpset operations (will reappear in odp_atomic_internal.h). Made some 64-bit operations (e.g. load, store) atomic on 32-bit architectures. odp_atomic_init_u32() and odp_atomic_init_u64() now take initialization value as parameter. odp_barrier.c: Use new odp_atomic_init_xxx() prototype. Use odp_atomic.h functions to access atomic variables. Fixed non-atomic wraparound (using new odp_atomic_sub_u32() function). odp_generator.c: Use new odp_atomic_init_xxx() prototype. odp_ticketlock.c: Use new odp_atomic_init_xxx() prototype. odp_atomic_text.c: Use new odp_atomic_init_xxx() prototype. example/generator/odp_generator.c | 8 +- platform/linux-generic/include/api/odp_atomic.h | 271 +++++++++++++++--------- platform/linux-generic/odp_barrier.c | 13 +- platform/linux-generic/odp_ticketlock.c | 3 +- test/api_test/odp_atomic_test.c | 4 +- 5 files changed, 181 insertions(+), 118 deletions(-)