All Data Structures Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
modules/common/target/Linux-x86_64/include/citrusleaf/cf_atomic.h
Go to the documentation of this file.
1 /******************************************************************************
2  * Copyright 2008-2013 by Aerospike.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a copy
5  * of this software and associated documentation files (the "Software"), to
6  * deal in the Software without restriction, including without limitation the
7  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
8  * sell copies of the Software, and to permit persons to whom the Software is
9  * furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
20  * IN THE SOFTWARE.
21  *****************************************************************************/
22 #pragma once
23 
24 /**
25  * SYNOPSIS
26  * Atomic memory operations
27  * Memory barriers
28  *
29  * cf_atomicX_add
30  * Atomic addition: add a value b into an atomic integer a, and return the result
31  *
32  * cf_atomicX_cas
33  * Compare-and-swap: test a value b against an atomic integer a; if they
34  * are equal, store the value x into a, and return the initial value of a.
35  * "Success" can be checked by comparing the returned value against b
36  * NB: this is a strong memory barrier
37  *
38  * cf_atomicX_fas
39  * Fetch-and-swap: swap the values of b and a
40  *
41  * cf_atomicX_addunless
42  * Increment-unless: test a value b against an atomic integer a. If they
43  * are NOT equal, add x to a, and return non-zero; if they ARE equal, return
44  * zero
45  **/
46 
47 #include <stdint.h>
48 #include <citrusleaf/cf_arch.h>
49 
50 #ifdef CF_WINDOWS
51 #include <intrin.h>
52 #include <WinSock2.h> // LONGLONG
53 #endif
54 
55 #ifdef __cplusplus
56 extern "C" {
57 #endif
58 
59 
60 #ifdef MARCH_i686
61 #define SIZEOF_ATOMIC_INT 4
62 typedef volatile uint32_t cf_atomic32;
63 typedef volatile uint32_t cf_atomic_p;
64 typedef volatile uint32_t cf_atomic_int;
65 typedef uint32_t cf_atomic_int_t; // the point here is a type of the same size as cf_atomic_int but isn't atomic
66 #endif
67 
68 #ifdef MARCH_x86_64
69 #define SIZEOF_ATOMIC_INT 8
70 typedef volatile uint64_t cf_atomic64;
71 typedef volatile uint32_t cf_atomic32;
72 typedef volatile uint64_t cf_atomic_p;
73 typedef volatile uint64_t cf_atomic_int;
74 typedef uint64_t cf_atomic_int_t; // the point here is a type of the same size as cf_atomic_int but isn't atomic
75 #endif
76 
77 
78 /******************************************************************************
79  * MACROS
80  *****************************************************************************/
81 
82 #define cf_atomic32_get(a) (a)
83 #define cf_atomic32_set(a, b) (*(a) = (b))
84 #define cf_atomic32_sub(a,b) (cf_atomic32_add((a), (0 - (b))))
85 #define cf_atomic32_incr(a) (cf_atomic32_add((a), 1))
86 #define cf_atomic32_decr(a) (cf_atomic32_add((a), -1))
87 
88 
89 #ifdef MARCH_x86_64
90 
91 #define cf_atomic64_get(a) (a)
92 #define cf_atomic64_set(a, b) (*(a) = (b))
93 #define cf_atomic64_sub(a,b) (cf_atomic64_add((a), (0 - (b))))
94 #define cf_atomic64_incr(a) (cf_atomic64_add((a), 1))
95 #define cf_atomic64_decr(a) (cf_atomic64_add((a), -1))
96 
97 #define cf_atomic_p_get(_a) cf_atomic64_get(_a)
98 #define cf_atomic_p_set(_a, _b) cf_atomic64_set(_a, _b)
99 #define cf_atomic_p_add(_a, _b) cf_atomic64_add(_a, _b)
100 #define cf_atomic_p_incr(_a) cf_atomic64_add((_a), 1)
101 #define cf_atomic_p_decr(_a) cf_atomic64_add((_a), -1)
102 
103 #define cf_atomic_int_get(_a) cf_atomic64_get(_a)
104 #define cf_atomic_int_set(_a, _b) cf_atomic64_set(_a, _b)
105 #define cf_atomic_int_add(_a, _b) cf_atomic64_add(_a, _b)
106 #define cf_atomic_int_sub(_a, _b) cf_atomic64_sub(_a, _b)
107 #define cf_atomic_int_incr(_a) cf_atomic64_add((_a), 1)
108 #define cf_atomic_int_decr(_a) cf_atomic64_add((_a), -1)
109 
110 #else // ifndef MARCH_x86_64
111 
112 #define cf_atomic_p_get(_a) cf_atomic32_get(_a)
113 #define cf_atomic_p_set(_a, _b) cf_atomic32_set(_a, _b)
114 #define cf_atomic_p_add(_a, _b) cf_atomic32_add(_a, _b)
115 #define cf_atomic_p_incr(_a) cf_atomic32_add((_a), 1)
116 #define cf_atomic_p_decr(_a) cf_atomic32_add((_a), -1)
117 
118 #define cf_atomic_int_get(_a) cf_atomic32_get(_a)
119 #define cf_atomic_int_set(_a, _b) cf_atomic32_set(_a, _b)
120 #define cf_atomic_int_add(_a, _b) cf_atomic32_add(_a, _b)
121 #define cf_atomic_int_sub(_a, _b) cf_atomic32_sub(_a, _b)
122 #define cf_atomic_int_incr(_a) cf_atomic32_add((_a), 1)
123 #define cf_atomic_int_decr(_a) cf_atomic32_add((_a), -1)
124 
125 #endif // ifdef MARCH_x86_64
126 
127 
128 #ifndef CF_WINDOWS
129 #ifdef MARCH_x86_64
130 
131 #define cf_atomic_p_cas(_a, _b, _x) cf_atomic64_cas(_a, _b, _x)
132 #define cf_atomic_p_cas_m(_a, _b, _x) cf_atomic64_cas_m(_a, _b, _x)
133 #define cf_atomic_p_fas(_a, _b) cf_atomic64_fas(_a, _b)
134 #define cf_atomic_p_fas_m(_a, _b) cf_atomic64_fas_m(_a, _b)
135 #define cf_atomic_p_addunless(_a, _b, _x) cf_atomic64_addunless(_a, _b, _x)
136 #define cf_atomic_p_setmax(_a, _x) cf_atomic64_setmax(_a, _x)
137 
138 #define cf_atomic_int_cas(_a, _b, _x) cf_atomic64_cas(_a, _b, _x)
139 #define cf_atomic_int_cas_m(_a, _b, _x) cf_atomic64_cas_m(_a, _b, _x)
140 #define cf_atomic_int_fas(_a, _b) cf_atomic64_fas(_a, _b)
141 #define cf_atomic_int_fas_m(_a, _b) cf_atomic64_fas_m(_a, _b)
142 #define cf_atomic_int_addunless(_a, _b, _x) cf_atomic64_addunless(_a, _b, _x)
143 #define cf_atomic_int_setmax(_a, _x) cf_atomic64_setmax(_a, _x)
144 
145 #else // ifndef CF_WINDOWS && ifndef MARCH_x86_64
146 
147 #define cf_atomic_p_cas(_a, _b, _x) cf_atomic32_cas(_a, _b, _x)
148 #define cf_atomic_p_cas_m(_a, _b, _x) cf_atomic32_cas_m(_a, _b, _x)
149 #define cf_atomic_p_fas(_a, _b) cf_atomic32_fas(_a, _b)
150 #define cf_atomic_p_fas_m(_a, _b) cf_atomic32_fas_m(_a, _b)
151 #define cf_atomic_p_addunless(_a, _b, _x) cf_atomic32_addunless(_a, _b, _x)
152 #define cf_atomic_p_setmax(_a, _x) cf_atomic32_setmax(_a, _x)
153 
154 #define cf_atomic_int_cas(_a, _b, _x) cf_atomic32_cas(_a, _b, _x)
155 #define cf_atomic_int_cas_m(_a, _b, _x) cf_atomic32_cas_m(_a, _b, _x)
156 #define cf_atomic_int_fas(_a, _b) cf_atomic32_fas(_a, _b)
157 #define cf_atomic_int_fas_m(_a, _b) cf_atomic32_fas_m(_a, _b)
158 #define cf_atomic_int_addunless(_a, _b, _x) cf_atomic32_addunless(_a, _b, _x)
159 #define cf_atomic_int_setmax(_a, _x) cf_atomic32_setmax(_a, _x)
160 
161 #endif // ifdef MARCH_x86_64
162 #endif // ifndef CF_WINDOWS
163 
164 
165 #ifdef CF_WINDOWS
166 
167 #define smb_mb() _ReadWriteBarrier()
168 
169 #else // ifndef CF_WINDOWS
170 
171 #define smb_mb() asm volatile("mfence":::"memory")
172 
173 /* CF_BARRIER
174  * All preceding memory accesses must commit before any following accesses */
175 #define CF_MEMORY_BARRIER() __asm__ __volatile__ ("lock; addl $0,0(%%esp)" : : : "memory")
176 
177 /* CF_BARRIER_READ
178  * All preceding memory accesses must commit before any following accesses */
179 #define CF_MEMORY_BARRIER_READ() CF_MEMORY_BARRIER()
180 
181 /* CF_BARRIER_WRITE
182  * All preceding memory accesses must commit before any following accesses */
183 #define CF_MEMORY_BARRIER_WRITE() __asm__ __volatile__ ("" : : : "memory")
184 
185 #endif // CF_WINDOWS
186 
187 /******************************************************************************
188  * FUNCTIONS
189  *****************************************************************************/
190 
191 static inline int64_t cf_atomic64_add(cf_atomic64 *, int64_t);
192 static inline int32_t cf_atomic32_add(cf_atomic32 *, int32_t);
193 
194 /******************************************************************************
195  * 64-BIT WINDOWS FUNCTIONS
196  *****************************************************************************/
197 
198 #ifdef CF_WINDOWS
199 #ifdef MARCH_x86_64
200 
201 static inline int64_t cf_atomic64_add(cf_atomic64 *a, int64_t b) {
202  int64_t i = b;
203  b = _InterlockedExchangeAdd64((LONGLONG *)a, b);
204  return(b + i);
205 }
206 
207 #endif // ifdef MARCH_x86_64
208 #endif // ifdef CF_WINDOWS
209 
210 /******************************************************************************
211  * 64-BIT LINUX FUNCTIONS
212  *****************************************************************************/
213 
214 #ifndef CF_WINDOWS
215 #ifdef MARCH_x86_64
216 
217 static inline int64_t cf_atomic64_add(cf_atomic64 *a, int64_t b) {
218  int64_t i = b;
219  __asm__ __volatile__ ("lock; xaddq %0, %1" : "+r" (b), "+m" (*a) : : "memory");
220  return(b + i);
221 }
222 
223 #define cf_atomic64_cas_m(_a, _b, _x) ({ \
224  __typeof__(_b) __b = _b; \
225  __asm__ __volatile__ ("lock; cmpxchgq %1,%2" : "=a"(__b) : "q"(_x), "m"(*(_a)), "0"(_b) : "memory"); \
226  __b; \
227 })
228 
229 #define cf_atomic64_fas_m(_a, _b) ({ \
230  __typeof__(_b) __b; \
231  __asm__ __volatile__ ("lock; xchgq %0,%1" : "=r"(__b) : "m"(*(_a)), "0"(_b)); \
232  __b; \
233 })
234 
235 static inline int64_t cf_atomic64_cas(cf_atomic64 *a, int64_t b, int64_t x) {
236  int64_t p;
237  __asm__ __volatile__ ("lock; cmpxchgq %1,%2" : "=a"(p) : "q"(x), "m"(*(a)), "0"(b) : "memory");
238  return(p);
239 }
240 
241 
242 static inline int64_t cf_atomic64_fas(cf_atomic64 *a, cf_atomic64 *b) {
243  int64_t p;
244  __asm__ __volatile__ ("lock; xchgq %0,%1" : "=r"(p) : "m"(*(a)), "0"(*(b)) : "memory");
245  return(p);
246 }
247 
248 static inline int64_t cf_atomic64_addunless(cf_atomic64 *a, int64_t b, int64_t x) {
249  int64_t prior, cur;
250 
251  // Get the current value of the atomic integer
252  cur = cf_atomic64_get(*a);
253 
254  for ( ;; ) {
255  // Check if the current value is equal to the criterion
256  if (cur == b)
257  break;
258 
259  // Attempt a compare-and-swap, which will return the value of cur
260  // prior to the operation
261  prior = cf_atomic64_cas(a, cur, cur + x);
262 
263  // If prior and cur are equal, then the operation has succeeded;
264  // otherwise, set cur to prior and go around again
265  if (prior == cur)
266  break;
267  cur = prior;
268  }
269 
270  return(cur != b);
271 }
272 
273 static inline int64_t cf_atomic64_setmax(cf_atomic64 *a, int64_t x) {
274  int64_t prior, cur;
275 
276  /* Get the current value of the atomic integer */
277  cur = cf_atomic64_get(*a);
278 
279  for ( ;; ) {
280  /* Check if the current value is equal to the criterion */
281  if (cur >= x)
282  break;
283 
284  /* Attempt a compare-and-swap, which will return the value of cur
285  * prior to the operation */
286  prior = cf_atomic64_cas(a, cur, x);
287 
288  /* If prior and cur are equal, then the operation has succeeded;
289  * otherwise, set cur to prior and go around again */
290  if (prior == cur)
291  break;
292  cur = prior;
293  }
294 
295  return(cur != x);
296 }
297 
298 #endif // ifdef MARCH_x86_64
299 #endif // ifndef CF_WINDOWS
300 
301 /******************************************************************************
302  * 32-BIT WINDOWS FUNCTIONS
303  *****************************************************************************/
304 
305 #ifdef CF_WINDOWS
306 
307 static inline int32_t cf_atomic32_add(cf_atomic32 *a, int32_t b){
308  int32_t i = b;
309  b = _InterlockedExchangeAdd((volatile long *)a, b);
310  return(b + i);
311 }
312 
313 #endif // ifdef CF_WINDOWS
314 
315 /******************************************************************************
316  * 32-BIT LINUX FUNCTIONS
317  *****************************************************************************/
318 
319 #ifndef CF_WINDOWS
320 
321 static inline int32_t cf_atomic32_add(cf_atomic32 *a, int32_t b){
322  int32_t i = b;
323  __asm__ __volatile__ ("lock; xadd %0, %1" : "+r" (b), "+m" (*a) : : "memory");
324  return(b + i);
325 }
326 
327 #define cf_atomic32_fas_m(_a, _b) ({ \
328  __typeof__(_b) __b; \
329  __asm__ __volatile__ ("lock; xchg %0,%1" : "=r"(__b) : "m"(*(_a)), "0"(_b)); \
330  __b; \
331 })
332 
333 #define cf_atomic32_cas_m(_a, _b, _x) \ ({ \
334  __typeof__(_b) __b = _b; \
335  __asm__ __volatile__ ("lock; cmpxchg %1,%2" : "=a"(__b) : "q"(_x), "m"(*(_a)), "0"(_b) : "memory"); \
336  __b; \
337 })
338 
339 static inline int32_t cf_atomic32_cas(cf_atomic32 *a, int32_t b, int32_t x) {
340  int32_t p;
341  __asm__ __volatile__ ("lock; cmpxchg %1,%2" : "=a"(p) : "q"(x), "m"(*(a)), "0"(b) : "memory");
342  return(p);
343 }
344 
345 static inline int32_t cf_atomic32_fas(cf_atomic32 *a, cf_atomic32 *b) {
346  int32_t p;
347  __asm__ __volatile__ ("lock; xchg %0,%1" : "=r"(p) : "m"(*(a)), "0"(*(b)) : "memory");
348  return(p);
349 }
350 
351 static inline int32_t cf_atomic32_addunless(cf_atomic32 *a, int32_t b, int32_t x) {
352  int32_t prior, cur;
353 
354  // Get the current value of the atomic integer
355  cur = cf_atomic32_get(*a);
356 
357  for ( ;; ) {
358  // Check if the current value is equal to the criterion
359  if (cur == b) break;
360 
361  // Attempt a compare-and-swap, which will return the value of cur
362  // prior to the operation
363  prior = cf_atomic32_cas(a, cur, cur + x);
364 
365  // If prior and cur are equal, then the operation has succeeded;
366  // otherwise, set cur to prior and go around again
367  if (prior == cur) break;
368 
369  cur = prior;
370  }
371  return(cur != b);
372 }
373 
374 static inline int32_t cf_atomic32_setmax(cf_atomic32 *a, int32_t x) {
375  int32_t prior, cur;
376 
377  // Get the current value of the atomic integer
378  cur = cf_atomic32_get(*a);
379 
380  for ( ;; ) {
381  // Check if the current value is equal to the criterion
382  if (cur >= x) break;
383 
384  // Attempt a compare-and-swap, which will return the value of cur
385  // prior to the operation
386  prior = cf_atomic32_cas(a, cur, x);
387 
388  // If prior and cur are equal, then the operation has succeeded;
389  // otherwise, set cur to prior and go around again
390  if (prior == cur) break;
391 
392  cur = prior;
393  }
394 
395  return(cur != x);
396 }
397 
398 #endif // ifndef CF_WINDOWS
399 
400 /*****************************************************************************/
401 
402 #ifdef __cplusplus
403 } // end extern "C"
404 #endif