46 #ifndef OPAL_SYS_ATOMIC_H
47 #define OPAL_SYS_ATOMIC_H 1
49 #include "opal_config.h"
51 #include "opal/sys/architecture.h"
53 #ifdef HAVE_SYS_TYPES_H
54 #include <sys/types.h>
59 #ifdef OMPI_DISABLE_INLINE_ASM
60 #undef OPAL_C_GCC_INLINE_ASSEMBLY
61 #define OPAL_C_GCC_INLINE_ASSEMBLY 0
62 #undef OMPI_CXX_GCC_INLINE_ASSEMBLY
63 #define OMPI_CXX_GCC_INLINE_ASSEMBLY 0
64 #undef OPAL_C_DEC_INLINE_ASSEMBLY
65 #define OPAL_C_DEC_INLINE_ASSEMBLY 0
66 #undef OMPI_CXX_DEC_INLINE_ASSEMBLY
67 #define OMPI_CXX_DEC_INLINE_ASSEMBLY 0
68 #undef OPAL_C_XLC_INLINE_ASSEMBLY
69 #define OPAL_C_XLC_INLINE_ASSEMBLY 0
70 #undef OMPI_CXX_XLC_INLINE_ASSEMBLY
71 #define OMPI_CXX_XLC_INLINE_ASSEMBLY 0
77 #if defined(c_plusplus) || defined(__cplusplus)
78 #define OMPI_GCC_INLINE_ASSEMBLY OMPI_CXX_GCC_INLINE_ASSEMBLY
79 #define OMPI_DEC_INLINE_ASSEMBLY OMPI_CXX_DEC_INLINE_ASSEMBLY
80 #define OMPI_XLC_INLINE_ASSEMBLY OMPI_CXX_XLC_INLINE_ASSEMBLY
82 #define OMPI_GCC_INLINE_ASSEMBLY OPAL_C_GCC_INLINE_ASSEMBLY
83 #define OMPI_DEC_INLINE_ASSEMBLY OPAL_C_DEC_INLINE_ASSEMBLY
84 #define OMPI_XLC_INLINE_ASSEMBLY OPAL_C_XLC_INLINE_ASSEMBLY
117 #if !OMPI_GCC_INLINE_ASSEMBLY
118 #define OPAL_HAVE_INLINE_ATOMIC_MEM_BARRIER 0
119 #define OPAL_HAVE_INLINE_ATOMIC_CMPSET_32 0
120 #define OPAL_HAVE_INLINE_ATOMIC_CMPSET_64 0
121 #define OPAL_HAVE_INLINE_ATOMIC_ADD_32 0
122 #define OPAL_HAVE_INLINE_ATOMIC_SUB_32 0
123 #define OPAL_HAVE_INLINE_ATOMIC_ADD_64 0
124 #define OPAL_HAVE_INLINE_ATOMIC_SUB_64 0
125 #define OPAL_HAVE_INLINE_ATOMIC_SWAP_32 0
126 #define OPAL_HAVE_INLINE_ATOMIC_SWAP_64 0
128 #define OPAL_HAVE_INLINE_ATOMIC_MEM_BARRIER 1
129 #define OPAL_HAVE_INLINE_ATOMIC_CMPSET_32 1
130 #define OPAL_HAVE_INLINE_ATOMIC_CMPSET_64 1
131 #define OPAL_HAVE_INLINE_ATOMIC_ADD_32 1
132 #define OPAL_HAVE_INLINE_ATOMIC_SUB_32 1
133 #define OPAL_HAVE_INLINE_ATOMIC_ADD_64 1
134 #define OPAL_HAVE_INLINE_ATOMIC_SUB_64 1
135 #define OPAL_HAVE_INLINE_ATOMIC_SWAP_32 1
136 #define OPAL_HAVE_INLINE_ATOMIC_SWAP_64 1
147 #elif OPAL_ASSEMBLY_ARCH == OMPI_WINDOWS
149 #include "opal/sys/win32/atomic.h"
150 #elif OPAL_ASSEMBLY_ARCH == OMPI_ALPHA
151 #include "opal/sys/alpha/atomic.h"
152 #elif OPAL_ASSEMBLY_ARCH == OMPI_AMD64
153 #include "opal/sys/amd64/atomic.h"
154 #elif OPAL_ASSEMBLY_ARCH == OMPI_ARM
155 #include "opal/sys/arm/atomic.h"
156 #elif OPAL_ASSEMBLY_ARCH == OMPI_IA32
157 #include "opal/sys/ia32/atomic.h"
158 #elif OPAL_ASSEMBLY_ARCH == OMPI_IA64
159 #include "opal/sys/ia64/atomic.h"
160 #elif OPAL_ASSEMBLY_ARCH == OMPI_MIPS
161 #include "opal/sys/mips/atomic.h"
162 #elif OPAL_ASSEMBLY_ARCH == OMPI_POWERPC32
163 #include "opal/sys/powerpc/atomic.h"
164 #elif OPAL_ASSEMBLY_ARCH == OMPI_POWERPC64
165 #include "opal/sys/powerpc/atomic.h"
166 #elif OPAL_ASSEMBLY_ARCH == OMPI_SPARC
167 #include "opal/sys/sparc/atomic.h"
168 #elif OPAL_ASSEMBLY_ARCH == OMPI_SPARCV9_32
169 #include "opal/sys/sparcv9/atomic.h"
170 #elif OPAL_ASSEMBLY_ARCH == OMPI_SPARCV9_64
171 #include "opal/sys/sparcv9/atomic.h"
172 #elif OPAL_ASSEMBLY_ARCH == OMPI_SYNC_BUILTIN
173 #include "opal/sys/sync_builtin/atomic.h"
180 #ifndef OPAL_HAVE_ATOMIC_CMPSET_32
181 #define OPAL_HAVE_ATOMIC_CMPSET_32 0
183 #ifndef OPAL_HAVE_ATOMIC_CMPSET_64
184 #define OPAL_HAVE_ATOMIC_CMPSET_64 0
194 #if !defined(OPAL_HAVE_ATOMIC_MEM_BARRIER) && !defined(DOXYGEN)
196 #define OPAL_HAVE_ATOMIC_MEM_BARRIER 0
199 #if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MEM_BARRIER
214 #if OPAL_HAVE_INLINE_ATOMIC_MEM_BARRIER
229 #if OPAL_HAVE_INLINE_ATOMIC_MEM_BARRIER
244 #if OPAL_HAVE_INLINE_ATOMIC_MEM_BARRIER
258 #if !defined(OPAL_HAVE_ATOMIC_SPINLOCKS) && !defined(DOXYGEN)
261 #define OPAL_HAVE_ATOMIC_SPINLOCKS 0
264 #if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_SPINLOCKS || (OPAL_HAVE_ATOMIC_CMPSET_32 || OPAL_HAVE_ATOMIC_CMPSET_64)
270 OPAL_ATOMIC_UNLOCKED = 0,
271 OPAL_ATOMIC_LOCKED = 1
281 #if OPAL_HAVE_ATOMIC_SPINLOCKS == 0
293 #if OPAL_HAVE_ATOMIC_SPINLOCKS == 0
304 #if OPAL_HAVE_ATOMIC_SPINLOCKS == 0
315 #if OPAL_HAVE_ATOMIC_SPINLOCKS == 0
321 #if OPAL_HAVE_ATOMIC_SPINLOCKS == 0
322 #undef OPAL_HAVE_ATOMIC_SPINLOCKS
323 #define OPAL_HAVE_ATOMIC_SPINLOCKS (OPAL_HAVE_ATOMIC_CMPSET_32 || OPAL_HAVE_ATOMIC_CMPSET_64)
324 #define OPAL_NEED_INLINE_ATOMIC_SPINLOCKS
335 #if !defined(OPAL_HAVE_ATOMIC_CMPSET_32) && !defined(DOXYGEN)
336 #define OPAL_HAVE_ATOMIC_CMPSET_32 0
338 #if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_CMPSET_32
340 #if OPAL_HAVE_INLINE_ATOMIC_CMPSET_32
343 int opal_atomic_cmpset_32(
volatile int32_t *addr, int32_t oldval,
346 #if OPAL_HAVE_INLINE_ATOMIC_CMPSET_32
349 int opal_atomic_cmpset_acq_32(
volatile int32_t *addr, int32_t oldval,
352 #if OPAL_HAVE_INLINE_ATOMIC_CMPSET_32
355 int opal_atomic_cmpset_rel_32(
volatile int32_t *addr, int32_t oldval,
360 #if !defined(OPAL_HAVE_ATOMIC_CMPSET_64) && !defined(DOXYGEN)
361 #define OPAL_HAVE_ATOMIC_CMPSET_64 0
363 #if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_CMPSET_64
365 #if OPAL_HAVE_INLINE_ATOMIC_CMPSET_64
368 int opal_atomic_cmpset_64(
volatile int64_t *addr, int64_t oldval,
371 #if OPAL_HAVE_INLINE_ATOMIC_CMPSET_64
374 int opal_atomic_cmpset_acq_64(
volatile int64_t *addr, int64_t oldval,
377 #if OPAL_HAVE_INLINE_ATOMIC_CMPSET_64
380 int opal_atomic_cmpset_rel_64(
volatile int64_t *addr, int64_t oldval,
385 #if !defined(OPAL_HAVE_ATOMIC_MATH_32) && !defined(DOXYGEN)
387 #define OPAL_HAVE_ATOMIC_MATH_32 0
390 #if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_CMPSET_32
397 #if OPAL_HAVE_INLINE_ATOMIC_ADD_32 || OPAL_HAVE_ATOMIC_CMPSET_32
400 int32_t opal_atomic_add_32(
volatile int32_t *addr,
int delta);
407 #if OPAL_HAVE_INLINE_ATOMIC_SUB_32 || OPAL_HAVE_ATOMIC_CMPSET_32
410 int32_t opal_atomic_sub_32(
volatile int32_t *addr,
int delta);
414 #if ! OPAL_HAVE_ATOMIC_MATH_32
416 #undef OPAL_HAVE_ATOMIC_MATH_32
417 #define OPAL_HAVE_ATOMIC_MATH_32 OPAL_HAVE_ATOMIC_CMPSET_32
420 #ifndef OPAL_HAVE_ATOMIC_MATH_64
422 #define OPAL_HAVE_ATOMIC_MATH_64 0
425 #if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_64 || OPAL_HAVE_ATOMIC_CMPSET_64
432 #if OPAL_HAVE_INLINE_ATOMIC_ADD_64 || OPAL_HAVE_ATOMIC_CMPSET_64
435 int64_t opal_atomic_add_64(
volatile int64_t *addr, int64_t delta);
442 #if OPAL_HAVE_INLINE_ATOMIC_SUB_64 || OPAL_HAVE_ATOMIC_CMPSET_64
445 int64_t opal_atomic_sub_64(
volatile int64_t *addr, int64_t delta);
449 #if ! OPAL_HAVE_ATOMIC_MATH_64
451 #undef OPAL_HAVE_ATOMIC_MATH_64
452 #define OPAL_HAVE_ATOMIC_MATH_64 OPAL_HAVE_ATOMIC_CMPSET_64
460 #if defined(DOXYGEN) || OPAL_ENABLE_DEBUG
462 opal_atomic_add_size_t(
volatile size_t *addr,
int delta)
464 #if SIZEOF_SIZE_T == 4
465 return (
size_t) opal_atomic_add_32((int32_t*) addr, delta);
466 #elif SIZEOF_SIZE_T == 8
467 return (
size_t) opal_atomic_add_64((int64_t*) addr, delta);
469 #error "Unknown size_t size"
473 opal_atomic_sub_size_t(
volatile size_t *addr,
int delta)
475 #if SIZEOF_SIZE_T == 4
476 return (
size_t) opal_atomic_sub_32((int32_t*) addr, delta);
477 #elif SIZEOF_SIZE_T == 8
478 return (
size_t) opal_atomic_sub_64((int64_t*) addr, delta);
480 #error "Unknown size_t size"
484 #if SIZEOF_SIZE_T == 4
485 #define opal_atomic_add_size_t(addr, delta) ((size_t) opal_atomic_add_32((int32_t*) addr, delta))
486 #define opal_atomic_sub_size_t(addr, delta) ((size_t) opal_atomic_sub_32((int32_t*) addr, delta))
487 #elif SIZEOF_SIZE_T ==8
488 #define opal_atomic_add_size_t(addr, delta) ((size_t) opal_atomic_add_64((int64_t*) addr, delta))
489 #define opal_atomic_sub_size_t(addr, delta) ((size_t) opal_atomic_sub_64((int64_t*) addr, delta))
491 #error "Unknown size_t size"
495 #if defined(DOXYGEN) || (OPAL_HAVE_ATOMIC_CMPSET_32 || OPAL_HAVE_ATOMIC_CMPSET_64)
498 static inline int opal_atomic_cmpset_xx(
volatile void* addr, int64_t oldval,
499 int64_t newval,
size_t length);
500 static inline int opal_atomic_cmpset_acq_xx(
volatile void* addr,
501 int64_t oldval, int64_t newval,
503 static inline int opal_atomic_cmpset_rel_xx(
volatile void* addr,
504 int64_t oldval, int64_t newval,
507 static inline int opal_atomic_cmpset_ptr(
volatile void* addr,
510 static inline int opal_atomic_cmpset_acq_ptr(
volatile void* addr,
513 static inline int opal_atomic_cmpset_rel_ptr(
volatile void* addr,
530 #define opal_atomic_cmpset( ADDR, OLDVAL, NEWVAL ) \
531 opal_atomic_cmpset_xx( (volatile void*)(ADDR), (intptr_t)(OLDVAL), \
532 (intptr_t)(NEWVAL), sizeof(*(ADDR)) )
547 #define opal_atomic_cmpset_acq( ADDR, OLDVAL, NEWVAL ) \
548 opal_atomic_cmpset_acq_xx( (volatile void*)(ADDR), (int64_t)(OLDVAL), \
549 (int64_t)(NEWVAL), sizeof(*(ADDR)) )
565 #define opal_atomic_cmpset_rel( ADDR, OLDVAL, NEWVAL ) \
566 opal_atomic_cmpset_rel_xx( (volatile void*)(ADDR), (int64_t)(OLDVAL), \
567 (int64_t)(NEWVAL), sizeof(*(ADDR)) )
571 #if defined(DOXYGEN) || (OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64)
573 static inline void opal_atomic_add_xx(
volatile void* addr,
574 int32_t value,
size_t length);
575 static inline void opal_atomic_sub_xx(
volatile void* addr,
576 int32_t value,
size_t length);
577 #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_CMPSET_32
578 static inline int32_t opal_atomic_add_ptr(
volatile void* addr,
void* delta );
579 static inline int32_t opal_atomic_sub_ptr(
volatile void* addr,
void* delta );
580 #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_CMPSET_64
581 static inline int64_t opal_atomic_add_ptr(
volatile void* addr,
void* delta );
582 static inline int64_t opal_atomic_sub_ptr(
volatile void* addr,
void* delta );
584 #error Atomic arithmetic on pointers not supported
597 #define opal_atomic_add( ADDR, VALUE ) \
598 opal_atomic_add_xx( (volatile void*)(ADDR), (int32_t)(VALUE), \
611 #define opal_atomic_sub( ADDR, VALUE ) \
612 opal_atomic_sub_xx( (volatile void*)(ADDR), (int32_t)(VALUE), \
624 #include "opal/sys/atomic_impl.h"
volatile unsigned char sparc_lock
The lock address on sparc.
Definition: atomic.h:105
void opal_atomic_rmb(void)
Read memory barrier.
Volatile lock object (with optional padding).
Definition: atomic.h:102
static void opal_atomic_unlock(opal_atomic_lock_t *lock)
Release a lock.
void opal_atomic_mb(void)
Memory barrier.
static void opal_atomic_lock(opal_atomic_lock_t *lock)
Acquire a lock by spinning.
void opal_atomic_wmb(void)
Write memory barrier.
static void opal_atomic_init(opal_atomic_lock_t *lock, int32_t value)
Initialize a lock to value.
volatile int32_t lock
The lock address (an integer)
Definition: atomic.h:104
static int opal_atomic_trylock(opal_atomic_lock_t *lock)
Try to acquire a lock.
char padding[sizeof(int)]
Array for optional padding.
Definition: atomic.h:106