OpenMPI  0.1.1
atomic.h
1 /*
2  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
3  * University Research and Technology
4  * Corporation. All rights reserved.
5  * Copyright (c) 2004-2005 The University of Tennessee and The University
6  * of Tennessee Research Foundation. All rights
7  * reserved.
8  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9  * University of Stuttgart. All rights reserved.
10  * Copyright (c) 2004-2005 The Regents of the University of California.
11  * All rights reserved.
12  * Copyright (c) 2010 IBM Corporation. All rights reserved.
13  * Copyright (c) 2010 ARM ltd. All rights reserved.
14  * $COPYRIGHT$
15  *
16  * Additional copyrights may follow
17  *
18  * $HEADER$
19  */
20 
21 #ifndef OMPI_SYS_ARCH_ATOMIC_H
22 #define OMPI_SYS_ARCH_ATOMIC_H 1
23 
24 #if OPAL_WANT_SMP_LOCKS
25 
26 #define MB() __asm__ __volatile__ ("dmb" : : : "memory")
27 #define RMB() __asm__ __volatile__ ("dmb" : : : "memory")
28 #define WMB() __asm__ __volatile__ ("dmb" : : : "memory")
29 
30 #else
31 
32 #define MB()
33 #define RMB()
34 #define WMB()
35 
36 #endif
37 
38 
39 /**********************************************************************
40  *
41  * Define constants for ARMv7
42  *
43  *********************************************************************/
44 #define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
45 
46 #define OPAL_HAVE_ATOMIC_CMPSET_32 1
47 
48 #define OPAL_HAVE_ATOMIC_CMPSET_64 1
49 
50 #define OPAL_HAVE_ATOMIC_MATH_32 1
51 #define OPAL_HAVE_ATOMIC_ADD_32 1
52 #define OPAL_HAVE_ATOMIC_SUB_32 1
53 
54 
55 /**********************************************************************
56  *
57  * Memory Barriers
58  *
59  *********************************************************************/
60 #if OMPI_GCC_INLINE_ASSEMBLY
61 
62 static inline
63 void opal_atomic_mb(void)
64 {
65  MB();
66 }
67 
68 
69 static inline
70 void opal_atomic_rmb(void)
71 {
72  RMB();
73 }
74 
75 
76 static inline
77 void opal_atomic_wmb(void)
78 {
79  WMB();
80 }
81 
82 
83 /**********************************************************************
84  *
85  * Atomic math operations
86  *
87  *********************************************************************/
88 
89 static inline int opal_atomic_cmpset_32(volatile int32_t *addr,
90  int32_t oldval, int32_t newval)
91 {
92  int32_t ret, tmp;
93 
94  __asm__ __volatile__ (
95  "1: ldrex %0, [%2] \n"
96  " cmp %0, %3 \n"
97  " bne 2f \n"
98  " strex %1, %4, [%2] \n"
99  " cmp %1, #0 \n"
100  " bne 1b \n"
101  "2: \n"
102 
103  : "=&r" (ret), "=&r" (tmp)
104  : "r" (addr), "r" (oldval), "r" (newval)
105  : "cc", "memory");
106 
107  return (ret == oldval);
108 }
109 
110 /* these two functions aren't inlined in the non-gcc case because then
111  there would be two function calls (since neither cmpset_32 nor
112  atomic_?mb can be inlined). Instead, we "inline" them by hand in
113  the assembly, meaning there is one function call overhead instead
114  of two */
115 static inline int opal_atomic_cmpset_acq_32(volatile int32_t *addr,
116  int32_t oldval, int32_t newval)
117 {
118  int rc;
119 
120  rc = opal_atomic_cmpset_32(addr, oldval, newval);
121  opal_atomic_rmb();
122 
123  return rc;
124 }
125 
126 
127 static inline int opal_atomic_cmpset_rel_32(volatile int32_t *addr,
128  int32_t oldval, int32_t newval)
129 {
130  opal_atomic_wmb();
131  return opal_atomic_cmpset_32(addr, oldval, newval);
132 }
133 
134 
135 static inline int opal_atomic_cmpset_64(volatile int64_t *addr,
136  int64_t oldval, int64_t newval)
137 {
138  int64_t ret;
139  int tmp;
140 
141 
142  __asm__ __volatile__ (
143  "1: ldrexd %0, %H0, [%2] \n"
144  " cmp %0, %3 \n"
145  " it eq \n"
146  " cmpeq %H0, %H3 \n"
147  " bne 2f \n"
148  " strexd %1, %4, %H4, [%2] \n"
149  " cmp %1, #0 \n"
150  " bne 1b \n"
151  "2: \n"
152 
153  : "=&r" (ret), "=&r" (tmp)
154  : "r" (addr), "r" (oldval), "r" (newval)
155  : "cc", "memory");
156 
157  return (ret == oldval);
158 }
159 
160 /* these two functions aren't inlined in the non-gcc case because then
161  there would be two function calls (since neither cmpset_64 nor
162  atomic_?mb can be inlined). Instead, we "inline" them by hand in
163  the assembly, meaning there is one function call overhead instead
164  of two */
165 static inline int opal_atomic_cmpset_acq_64(volatile int64_t *addr,
166  int64_t oldval, int64_t newval)
167 {
168  int rc;
169 
170  rc = opal_atomic_cmpset_64(addr, oldval, newval);
171  opal_atomic_rmb();
172 
173  return rc;
174 }
175 
176 
177 static inline int opal_atomic_cmpset_rel_64(volatile int64_t *addr,
178  int64_t oldval, int64_t newval)
179 {
180  opal_atomic_wmb();
181  return opal_atomic_cmpset_64(addr, oldval, newval);
182 }
183 
184 
185 static inline int32_t opal_atomic_add_32(volatile int32_t* v, int inc)
186 {
187  int32_t t;
188  int tmp;
189 
190  __asm__ __volatile__(
191  "1: ldrex %0, [%2] \n"
192  " add %0, %0, %3 \n"
193  " strex %1, %0, [%2] \n"
194  " cmp %1, #0 \n"
195  " bne 1b \n"
196 
197  : "=&r" (t), "=&r" (tmp)
198  : "r" (v), "r" (inc)
199  : "cc", "memory");
200 
201 
202  return t;
203 }
204 
205 
206 static inline int32_t opal_atomic_sub_32(volatile int32_t* v, int dec)
207 {
208  int32_t t;
209  int tmp;
210 
211  __asm__ __volatile__(
212  "1: ldrex %0, [%2] \n"
213  " sub %0, %0, %3 \n"
214  " strex %1, %0, [%2] \n"
215  " cmp %1, #0 \n"
216  " bne 1b \n"
217 
218  : "=&r" (t), "=&r" (tmp)
219  : "r" (v), "r" (dec)
220  : "cc", "memory");
221 
222  return t;
223 }
224 
225 
226 #endif /* OMPI_GCC_INLINE_ASSEMBLY */
227 
228 #endif /* ! OMPI_SYS_ARCH_ATOMIC_H */
void opal_atomic_rmb(void)
Read memory barrier.
void opal_atomic_mb(void)
Memory barrier.
void opal_atomic_wmb(void)
Write memory barrier.