20 #ifndef OMPI_SYS_ARCH_ATOMIC_H
21 #define OMPI_SYS_ARCH_ATOMIC_H 1
27 #if OPAL_WANT_SMP_LOCKS
29 #define MB() __asm__ __volatile__ ("sync" : : : "memory")
30 #define RMB() __asm__ __volatile__ ("lwsync" : : : "memory")
31 #define WMB() __asm__ __volatile__ ("eieio" : : : "memory")
32 #define SMP_SYNC "sync \n\t"
33 #define SMP_ISYNC "\n\tisync"
51 #define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
53 #define OPAL_HAVE_ATOMIC_CMPSET_32 1
55 #define OPAL_HAVE_ATOMIC_MATH_32 1
56 #define OPAL_HAVE_ATOMIC_ADD_32 1
57 #define OPAL_HAVE_ATOMIC_SUB_32 1
60 #if (OPAL_ASSEMBLY_ARCH == OMPI_POWERPC64) || OPAL_ASM_SUPPORT_64BIT
61 #define OPAL_HAVE_ATOMIC_CMPSET_64 1
70 #if OMPI_GCC_INLINE_ASSEMBLY
92 #elif OMPI_XLC_INLINE_ASSEMBLY
100 #undef OPAL_HAVE_INLINE_ATOMIC_MEM_BARRIER
101 #define OPAL_HAVE_INLINE_ATOMIC_MEM_BARRIER 0
103 #pragma mc_func opal_atomic_mb { "7c0004ac" }
104 #pragma reg_killed_by opal_atomic_mb
106 #pragma mc_func opal_atomic_rmb { "7c2004ac" }
107 #pragma reg_killed_by opal_atomic_rmb
109 #pragma mc_func opal_atomic_wmb { "7c0006ac" }
110 #pragma reg_killed_by opal_atomic_wmb
119 #if OMPI_GCC_INLINE_ASSEMBLY
121 static inline int opal_atomic_cmpset_32(
volatile int32_t *addr,
122 int32_t oldval, int32_t newval)
126 __asm__ __volatile__ (
127 "1: lwarx %0, 0, %2 \n\t"
128 " cmpw 0, %0, %3 \n\t"
130 " stwcx. %4, 0, %2 \n\t"
133 :
"=&r" (ret),
"=m" (*addr)
134 :
"r" (addr),
"r" (oldval),
"r" (newval),
"m" (*addr)
137 return (ret == oldval);
145 static inline int opal_atomic_cmpset_acq_32(
volatile int32_t *addr,
146 int32_t oldval, int32_t newval)
150 rc = opal_atomic_cmpset_32(addr, oldval, newval);
157 static inline int opal_atomic_cmpset_rel_32(
volatile int32_t *addr,
158 int32_t oldval, int32_t newval)
161 return opal_atomic_cmpset_32(addr, oldval, newval);
167 #if (OPAL_ASSEMBLY_ARCH == OMPI_POWERPC64)
169 #if OMPI_GCC_INLINE_ASSEMBLY
170 static inline int opal_atomic_cmpset_64(
volatile int64_t *addr,
171 int64_t oldval, int64_t newval)
175 __asm__ __volatile__ (
176 "1: ldarx %0, 0, %2 \n\t"
177 " cmpd 0, %0, %3 \n\t"
179 " stdcx. %4, 0, %2 \n\t"
182 :
"=&r" (ret),
"=m" (*addr)
183 :
"r" (addr),
"r" (oldval),
"r" (newval),
"m" (*addr)
186 return (ret == oldval);
194 static inline int opal_atomic_cmpset_acq_64(
volatile int64_t *addr,
195 int64_t oldval, int64_t newval)
199 rc = opal_atomic_cmpset_64(addr, oldval, newval);
206 static inline int opal_atomic_cmpset_rel_64(
volatile int64_t *addr,
207 int64_t oldval, int64_t newval)
210 return opal_atomic_cmpset_64(addr, oldval, newval);
215 #elif (OPAL_ASSEMBLY_ARCH == OMPI_POWERPC32) && OPAL_ASM_SUPPORT_64BIT
218 #define ll_low(x) *(((unsigned int*)&(x))+0)
219 #define ll_high(x) *(((unsigned int*)&(x))+1)
222 #if OMPI_GCC_INLINE_ASSEMBLY
224 static inline int opal_atomic_cmpset_64(
volatile int64_t *addr,
225 int64_t oldval, int64_t newval)
240 __asm__ __volatile__ (
243 "1: ldarx r9, 0, %1 \n\t"
244 " cmpd 0, r9, r4 \n\t"
246 " stdcx. r5, 0, %1 \n\t"
250 "subfic r9,r5,0 \n\t"
254 "m"(oldval),
"m"(newval)
255 :
"r4",
"r5",
"r9",
"cc",
"memory");
265 static inline int opal_atomic_cmpset_acq_64(
volatile int64_t *addr,
266 int64_t oldval, int64_t newval)
270 rc = opal_atomic_cmpset_64(addr, oldval, newval);
277 static inline int opal_atomic_cmpset_rel_64(
volatile int64_t *addr,
278 int64_t oldval, int64_t newval)
281 return opal_atomic_cmpset_64(addr, oldval, newval);
289 #if OMPI_GCC_INLINE_ASSEMBLY
291 static inline int32_t opal_atomic_add_32(
volatile int32_t* v,
int inc)
295 __asm__ __volatile__(
296 "1: lwarx %0, 0, %3 \n\t"
297 " add %0, %2, %0 \n\t"
298 " stwcx. %0, 0, %3 \n\t"
300 :
"=&r" (t),
"=m" (*v)
301 :
"r" (inc),
"r" (v),
"m" (*v)
308 static inline int32_t opal_atomic_sub_32(
volatile int32_t* v,
int dec)
312 __asm__ __volatile__(
313 "1: lwarx %0,0,%3 \n\t"
314 " subf %0,%2,%0 \n\t"
315 " stwcx. %0,0,%3 \n\t"
317 :
"=&r" (t),
"=m" (*v)
318 :
"r" (dec),
"r" (v),
"m" (*v)
void opal_atomic_rmb(void)
Read memory barrier.
void opal_atomic_mb(void)
Memory barrier.
void opal_atomic_wmb(void)
Write memory barrier.