[glibc.git] / include / atomic.h

/* Internal macros for atomic operations for GNU C Library.
   Copyright (C) 2002-2014 Free Software Foundation, Inc.
   This file is part of the GNU C Library.
   Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, see
   <http://www.gnu.org/licenses/>.  */

#ifndef _ATOMIC_H
#define _ATOMIC_H	1

/* This header defines three types of macros:

   - atomic arithmetic and logic operation on memory.  They all
     have the prefix "atomic_".

   - conditionally atomic operations of the same kinds.  These
     always behave identical but can be faster when atomicity
     is not really needed since only one thread has access to
     the memory location.  In that case the code is slower in
     the multi-thread case.  The interfaces have the prefix
     "catomic_".

   - support functions like barriers.  They also have the prefix
     "atomic_".

   Architectures must provide a few lowlevel macros (the compare
   and exchange definitions).  All others are optional.  They
   should only be provided if the architecture has specific
   support for the operation.

   As <atomic.h> macros are usually heavily nested and often use local
   variables to make sure side-effects are evaluated properly, use for
   macro local variables a per-macro unique prefix.  This file uses
   __atgN_ prefix where N is different in each macro.  */

#include <stdlib.h>

#include <bits/atomic.h>

/* Wrapper macros to call pre_NN_post (mem, ...) where NN is the
   bit width of *MEM.  The calling macro puts parens around MEM
   and following args.  */
#define __atomic_val_bysize(pre, post, mem, ...)			      \
  ({									      \
    __typeof (*mem) __atg1_result;					      \
    if (sizeof (*mem) == 1)						      \
      __atg1_result = pre##_8_##post (mem, __VA_ARGS__);		      \
    else if (sizeof (*mem) == 2)					      \
      __atg1_result = pre##_16_##post (mem, __VA_ARGS__);		      \
    else if (sizeof (*mem) == 4)					      \
      __atg1_result = pre##_32_##post (mem, __VA_ARGS__);		      \
    else if (sizeof (*mem) == 8)					      \
      __atg1_result = pre##_64_##post (mem, __VA_ARGS__);		      \
    else								      \
      abort ();								      \
    __atg1_result;							      \
  })
#define __atomic_bool_bysize(pre, post, mem, ...)			      \
  ({									      \
    int __atg2_result;							      \
    if (sizeof (*mem) == 1)						      \
      __atg2_result = pre##_8_##post (mem, __VA_ARGS__);		      \
    else if (sizeof (*mem) == 2)					      \
      __atg2_result = pre##_16_##post (mem, __VA_ARGS__);		      \
    else if (sizeof (*mem) == 4)					      \
      __atg2_result = pre##_32_##post (mem, __VA_ARGS__);		      \
    else if (sizeof (*mem) == 8)					      \
      __atg2_result = pre##_64_##post (mem, __VA_ARGS__);		      \
    else								      \
      abort ();								      \
    __atg2_result;							      \
  })


/* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL.
   Return the old *MEM value.  */
#if !defined atomic_compare_and_exchange_val_acq \
    && defined __arch_compare_and_exchange_val_32_acq
# define atomic_compare_and_exchange_val_acq(mem, newval, oldval) \
  __atomic_val_bysize (__arch_compare_and_exchange_val,acq,		      \
		       mem, newval, oldval)
#endif


#ifndef catomic_compare_and_exchange_val_acq
# ifdef __arch_c_compare_and_exchange_val_32_acq
#  define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \
  __atomic_val_bysize (__arch_c_compare_and_exchange_val,acq,		      \
		       mem, newval, oldval)
# else
#  define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \
  atomic_compare_and_exchange_val_acq (mem, newval, oldval)
# endif
#endif


#ifndef catomic_compare_and_exchange_val_rel
# ifndef atomic_compare_and_exchange_val_rel
#  define catomic_compare_and_exchange_val_rel(mem, newval, oldval)	      \
  catomic_compare_and_exchange_val_acq (mem, newval, oldval)
# else
#  define catomic_compare_and_exchange_val_rel(mem, newval, oldval)	      \
  atomic_compare_and_exchange_val_rel (mem, newval, oldval)
# endif
#endif


#ifndef atomic_compare_and_exchange_val_rel
# define atomic_compare_and_exchange_val_rel(mem, newval, oldval)	      \
  atomic_compare_and_exchange_val_acq (mem, newval, oldval)
#endif


/* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL.
   Return zero if *MEM was changed or non-zero if no exchange happened.  */
#ifndef atomic_compare_and_exchange_bool_acq
# ifdef __arch_compare_and_exchange_bool_32_acq
#  define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
  __atomic_bool_bysize (__arch_compare_and_exchange_bool,acq,		      \
		        mem, newval, oldval)
# else
#  define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
  ({ /* Cannot use __oldval here, because macros later in this file might     \
	call this macro with __oldval argument.	 */			      \
     __typeof (oldval) __atg3_old = (oldval);				      \
     atomic_compare_and_exchange_val_acq (mem, newval, __atg3_old)	      \
       != __atg3_old;							      \
  })
# endif
#endif


#ifndef catomic_compare_and_exchange_bool_acq
# ifdef __arch_c_compare_and_exchange_bool_32_acq
#  define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
  __atomic_bool_bysize (__arch_c_compare_and_exchange_bool,acq,		      \
		        mem, newval, oldval)
# else
#  define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
  ({ /* Cannot use __oldval here, because macros later in this file might     \
	call this macro with __oldval argument.	 */			      \
     __typeof (oldval) __atg4_old = (oldval);				      \
     catomic_compare_and_exchange_val_acq (mem, newval, __atg4_old)	      \
       != __atg4_old;							      \
  })
# endif
#endif


#ifndef catomic_compare_and_exchange_bool_rel
# ifndef atomic_compare_and_exchange_bool_rel
#  define catomic_compare_and_exchange_bool_rel(mem, newval, oldval)	      \
  catomic_compare_and_exchange_bool_acq (mem, newval, oldval)
# else
#  define catomic_compare_and_exchange_bool_rel(mem, newval, oldval)	      \
  atomic_compare_and_exchange_bool_rel (mem, newval, oldval)
# endif
#endif


#ifndef atomic_compare_and_exchange_bool_rel
# define atomic_compare_and_exchange_bool_rel(mem, newval, oldval) \
  atomic_compare_and_exchange_bool_acq (mem, newval, oldval)
#endif


/* Store NEWVALUE in *MEM and return the old value.  */
#ifndef atomic_exchange_acq
# define atomic_exchange_acq(mem, newvalue) \
  ({ __typeof (*(mem)) __atg5_oldval;					      \
     __typeof (mem) __atg5_memp = (mem);				      \
     __typeof (*(mem)) __atg5_value = (newvalue);			      \
									      \
     do									      \
       __atg5_oldval = *__atg5_memp;					      \
     while (__builtin_expect						      \
	    (atomic_compare_and_exchange_bool_acq (__atg5_memp, __atg5_value, \
						   __atg5_oldval), 0));	      \
									      \
     __atg5_oldval; })
#endif

#ifndef atomic_exchange_rel
# define atomic_exchange_rel(mem, newvalue) atomic_exchange_acq (mem, newvalue)
#endif


/* Add VALUE to *MEM and return the old value of *MEM.  */
#ifndef atomic_exchange_and_add_acq
# ifdef atomic_exchange_and_add
#  define atomic_exchange_and_add_acq(mem, value) \
  atomic_exchange_and_add (mem, value)
# else
#  define atomic_exchange_and_add_acq(mem, value) \
  ({ __typeof (*(mem)) __atg6_oldval;					      \
     __typeof (mem) __atg6_memp = (mem);				      \
     __typeof (*(mem)) __atg6_value = (value);				      \
									      \
     do									      \
       __atg6_oldval = *__atg6_memp;					      \
     while (__builtin_expect						      \
	    (atomic_compare_and_exchange_bool_acq (__atg6_memp,		      \
						   __atg6_oldval	      \
						   + __atg6_value,	      \
						   __atg6_oldval), 0));	      \
									      \
     __atg6_oldval; })
# endif
#endif

#ifndef atomic_exchange_and_add_rel
# define atomic_exchange_and_add_rel(mem, value) \
  atomic_exchange_and_add_acq(mem, value)
#endif

#ifndef atomic_exchange_and_add
# define atomic_exchange_and_add(mem, value) \
  atomic_exchange_and_add_acq(mem, value)
#endif

#ifndef catomic_exchange_and_add
# define catomic_exchange_and_add(mem, value) \
  ({ __typeof (*(mem)) __atg7_oldv;					      \
     __typeof (mem) __atg7_memp = (mem);				      \
     __typeof (*(mem)) __atg7_value = (value);				      \
									      \
     do									      \
       __atg7_oldv = *__atg7_memp;					      \
     while (__builtin_expect						      \
	    (catomic_compare_and_exchange_bool_acq (__atg7_memp,	      \
						    __atg7_oldv		      \
						    + __atg7_value,	      \
						    __atg7_oldv), 0));	      \
									      \
     __atg7_oldv; })
#endif


#ifndef atomic_max
# define atomic_max(mem, value) \
  do {									      \
    __typeof (*(mem)) __atg8_oldval;					      \
    __typeof (mem) __atg8_memp = (mem);					      \
    __typeof (*(mem)) __atg8_value = (value);				      \
    do {								      \
      __atg8_oldval = *__atg8_memp;					      \
      if (__atg8_oldval >= __atg8_value)				      \
	break;								      \
    } while (__builtin_expect						      \
	     (atomic_compare_and_exchange_bool_acq (__atg8_memp, __atg8_value,\
						    __atg8_oldval), 0));      \
  } while (0)
#endif


#ifndef catomic_max
# define catomic_max(mem, value) \
  do {									      \
    __typeof (*(mem)) __atg9_oldv;					      \
    __typeof (mem) __atg9_memp = (mem);					      \
    __typeof (*(mem)) __atg9_value = (value);				      \
    do {								      \
      __atg9_oldv = *__atg9_memp;					      \
      if (__atg9_oldv >= __atg9_value)					      \
	break;								      \
    } while (__builtin_expect						      \
	     (catomic_compare_and_exchange_bool_acq (__atg9_memp,	      \
						     __atg9_value,	      \
						     __atg9_oldv), 0));	      \
  } while (0)
#endif


#ifndef atomic_min
# define atomic_min(mem, value) \
  do {									      \
    __typeof (*(mem)) __atg10_oldval;					      \
    __typeof (mem) __atg10_memp = (mem);				      \
    __typeof (*(mem)) __atg10_value = (value);				      \
    do {								      \
      __atg10_oldval = *__atg10_memp;					      \
      if (__atg10_oldval <= __atg10_value)				      \
	break;								      \
    } while (__builtin_expect						      \
	     (atomic_compare_and_exchange_bool_acq (__atg10_memp,	      \
						    __atg10_value,	      \
						    __atg10_oldval), 0));     \
  } while (0)
#endif


#ifndef atomic_add
# define atomic_add(mem, value) (void) atomic_exchange_and_add ((mem), (value))
#endif


#ifndef catomic_add
# define catomic_add(mem, value) \
  (void) catomic_exchange_and_add ((mem), (value))
#endif


#ifndef atomic_increment
# define atomic_increment(mem) atomic_add ((mem), 1)
#endif


#ifndef catomic_increment
# define catomic_increment(mem) catomic_add ((mem), 1)
#endif


#ifndef atomic_increment_val
# define atomic_increment_val(mem) (atomic_exchange_and_add ((mem), 1) + 1)
#endif


#ifndef catomic_increment_val
# define catomic_increment_val(mem) (catomic_exchange_and_add ((mem), 1) + 1)
#endif


/* Add one to *MEM and return true iff it's now zero.  */
#ifndef atomic_increment_and_test
# define atomic_increment_and_test(mem) \
  (atomic_exchange_and_add ((mem), 1) + 1 == 0)
#endif


#ifndef atomic_decrement
# define atomic_decrement(mem) atomic_add ((mem), -1)
#endif


#ifndef catomic_decrement
# define catomic_decrement(mem) catomic_add ((mem), -1)
#endif


#ifndef atomic_decrement_val
# define atomic_decrement_val(mem) (atomic_exchange_and_add ((mem), -1) - 1)
#endif


#ifndef catomic_decrement_val
# define catomic_decrement_val(mem) (catomic_exchange_and_add ((mem), -1) - 1)
#endif


/* Subtract 1 from *MEM and return true iff it's now zero.  */
#ifndef atomic_decrement_and_test
# define atomic_decrement_and_test(mem) \
  (atomic_exchange_and_add ((mem), -1) == 1)
#endif


/* Decrement *MEM if it is > 0, and return the old value.  */
#ifndef atomic_decrement_if_positive
# define atomic_decrement_if_positive(mem) \
  ({ __typeof (*(mem)) __atg11_oldval;					      \
     __typeof (mem) __atg11_memp = (mem);				      \
									      \
     do									      \
       {								      \
	 __atg11_oldval = *__atg11_memp;				      \
	 if (__glibc_unlikely (__atg11_oldval <= 0))			      \
	   break;							      \
       }								      \
     while (__builtin_expect						      \
	    (atomic_compare_and_exchange_bool_acq (__atg11_memp,	      \
						   __atg11_oldval - 1,	      \
						   __atg11_oldval), 0));      \
     __atg11_oldval; })
#endif


#ifndef atomic_add_negative
# define atomic_add_negative(mem, value)				      \
  ({ __typeof (value) __atg12_value = (value);				      \
     atomic_exchange_and_add (mem, __atg12_value) < -__atg12_value; })
#endif


#ifndef atomic_add_zero
# define atomic_add_zero(mem, value)					      \
  ({ __typeof (value) __atg13_value = (value);				      \
     atomic_exchange_and_add (mem, __atg13_value) == -__atg13_value; })
#endif


#ifndef atomic_bit_set
# define atomic_bit_set(mem, bit) \
  (void) atomic_bit_test_set(mem, bit)
#endif


#ifndef atomic_bit_test_set
# define atomic_bit_test_set(mem, bit) \
  ({ __typeof (*(mem)) __atg14_old;					      \
     __typeof (mem) __atg14_memp = (mem);				      \
     __typeof (*(mem)) __atg14_mask = ((__typeof (*(mem))) 1 << (bit));	      \
									      \
     do									      \
       __atg14_old = (*__atg14_memp);					      \
     while (__builtin_expect						      \
	    (atomic_compare_and_exchange_bool_acq (__atg14_memp,	      \
						   __atg14_old | __atg14_mask,\
						   __atg14_old), 0));	      \
									      \
     __atg14_old & __atg14_mask; })
#endif

/* Atomically *mem &= mask.  */
#ifndef atomic_and
# define atomic_and(mem, mask) \
  do {									      \
    __typeof (*(mem)) __atg15_old;					      \
    __typeof (mem) __atg15_memp = (mem);				      \
    __typeof (*(mem)) __atg15_mask = (mask);				      \
									      \
    do									      \
      __atg15_old = (*__atg15_memp);					      \
    while (__builtin_expect						      \
	   (atomic_compare_and_exchange_bool_acq (__atg15_memp,		      \
						  __atg15_old & __atg15_mask, \
						  __atg15_old), 0));	      \
  } while (0)
#endif

#ifndef catomic_and
# define catomic_and(mem, mask) \
  do {									      \
    __typeof (*(mem)) __atg20_old;					      \
    __typeof (mem) __atg20_memp = (mem);				      \
    __typeof (*(mem)) __atg20_mask = (mask);				      \
									      \
    do									      \
      __atg20_old = (*__atg20_memp);					      \
    while (__builtin_expect						      \
	   (catomic_compare_and_exchange_bool_acq (__atg20_memp,	      \
						   __atg20_old & __atg20_mask,\
						   __atg20_old), 0));	      \
  } while (0)
#endif

/* Atomically *mem &= mask and return the old value of *mem.  */
#ifndef atomic_and_val
# define atomic_and_val(mem, mask) \
  ({ __typeof (*(mem)) __atg16_old;					      \
     __typeof (mem) __atg16_memp = (mem);				      \
     __typeof (*(mem)) __atg16_mask = (mask);				      \
									      \
     do									      \
       __atg16_old = (*__atg16_memp);					      \
     while (__builtin_expect						      \
	    (atomic_compare_and_exchange_bool_acq (__atg16_memp,	      \
						   __atg16_old & __atg16_mask,\
						   __atg16_old), 0));	      \
									      \
     __atg16_old; })
#endif

/* Atomically *mem |= mask and return the old value of *mem.  */
#ifndef atomic_or
# define atomic_or(mem, mask) \
  do {									      \
    __typeof (*(mem)) __atg17_old;					      \
    __typeof (mem) __atg17_memp = (mem);				      \
    __typeof (*(mem)) __atg17_mask = (mask);				      \
									      \
    do									      \
      __atg17_old = (*__atg17_memp);					      \
    while (__builtin_expect						      \
	   (atomic_compare_and_exchange_bool_acq (__atg17_memp,		      \
						  __atg17_old | __atg17_mask, \
						  __atg17_old), 0));	      \
  } while (0)
#endif

#ifndef catomic_or
# define catomic_or(mem, mask) \
  do {									      \
    __typeof (*(mem)) __atg18_old;					      \
    __typeof (mem) __atg18_memp = (mem);				      \
    __typeof (*(mem)) __atg18_mask = (mask);				      \
									      \
    do									      \
      __atg18_old = (*__atg18_memp);					      \
    while (__builtin_expect						      \
	   (catomic_compare_and_exchange_bool_acq (__atg18_memp,	      \
						   __atg18_old | __atg18_mask,\
						   __atg18_old), 0));	      \
  } while (0)
#endif

/* Atomically *mem |= mask and return the old value of *mem.  */
#ifndef atomic_or_val
# define atomic_or_val(mem, mask) \
  ({ __typeof (*(mem)) __atg19_old;					      \
     __typeof (mem) __atg19_memp = (mem);				      \
     __typeof (*(mem)) __atg19_mask = (mask);				      \
									      \
     do									      \
       __atg19_old = (*__atg19_memp);					      \
     while (__builtin_expect						      \
	    (atomic_compare_and_exchange_bool_acq (__atg19_memp,	      \
						   __atg19_old | __atg19_mask,\
						   __atg19_old), 0));	      \
									      \
     __atg19_old; })
#endif

#ifndef atomic_full_barrier
# define atomic_full_barrier() __asm ("" ::: "memory")
#endif


#ifndef atomic_read_barrier
# define atomic_read_barrier() atomic_full_barrier ()
#endif


#ifndef atomic_write_barrier
# define atomic_write_barrier() atomic_full_barrier ()
#endif


#ifndef atomic_forced_read
# define atomic_forced_read(x) \
  ({ __typeof (x) __x; __asm ("" : "=r" (__x) : "0" (x)); __x; })
#endif


#ifndef atomic_delay
# define atomic_delay() do { /* nothing */ } while (0)
#endif

#endif	/* atomic.h */
Commit	Line	Data
d15851ec	1	/* Internal macros for atomic operations for GNU C Library.
d4697bc9	2	Copyright (C) 2002-2014 Free Software Foundation, Inc.
76a50749 UD	3	This file is part of the GNU C Library.
	4	Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
	5
	6	The GNU C Library is free software; you can redistribute it and/or
	7	modify it under the terms of the GNU Lesser General Public
	8	License as published by the Free Software Foundation; either
	9	version 2.1 of the License, or (at your option) any later version.
	10
	11	The GNU C Library is distributed in the hope that it will be useful,
	12	but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	Lesser General Public License for more details.
	15
	16	You should have received a copy of the GNU Lesser General Public
59ba27a6 PE	17	License along with the GNU C Library; if not, see
59ba27a6 PE	18	<http://www.gnu.org/licenses/>. */
76a50749 UD	19
	20	#ifndef _ATOMIC_H
	21	#define _ATOMIC_H 1
	22
11bf311e UD	23	/* This header defines three types of macros:
	24
	25	- atomic arithmetic and logic operation on memory. They all
	26	have the prefix "atomic_".
	27
	28	- conditionally atomic operations of the same kinds. These
	29	always behave identical but can be faster when atomicity
	30	is not really needed since only one thread has access to
	31	the memory location. In that case the code is slower in
	32	the multi-thread case. The interfaces have the prefix
	33	"catomic_".
	34
eaea92f1	35	- support functions like barriers. They also have the prefix
11bf311e UD	36	"atomic_".
	37
	38	Architectures must provide a few lowlevel macros (the compare
	39	and exchange definitions). All others are optional. They
	40	should only be provided if the architecture has specific
	41	support for the operation.
	42
	43	As <atomic.h> macros are usually heavily nested and often use local
	44	variables to make sure side-effects are evaluated properly, use for
	45	macro local variables a per-macro unique prefix. This file uses
	46	__atgN_ prefix where N is different in each macro. */
	47
76a50749 UD	48	#include <stdlib.h>
	49
	50	#include <bits/atomic.h>
	51
fe4610b3 RM	52	/* Wrapper macros to call pre_NN_post (mem, ...) where NN is the
	53	bit width of *MEM. The calling macro puts parens around MEM
	54	and following args. */
	55	#define __atomic_val_bysize(pre, post, mem, ...) \
	56	({ \
11bf311e	57	__typeof (*mem) __atg1_result; \
fe4610b3	58	if (sizeof (*mem) == 1) \
11bf311e	59	__atg1_result = pre##_8_##post (mem, __VA_ARGS__); \
fe4610b3	60	else if (sizeof (*mem) == 2) \
11bf311e	61	__atg1_result = pre##_16_##post (mem, __VA_ARGS__); \
fe4610b3	62	else if (sizeof (*mem) == 4) \
11bf311e	63	__atg1_result = pre##_32_##post (mem, __VA_ARGS__); \
fe4610b3	64	else if (sizeof (*mem) == 8) \
11bf311e	65	__atg1_result = pre##_64_##post (mem, __VA_ARGS__); \
fe4610b3 RM	66	else \
fe4610b3 RM	67	abort (); \
11bf311e	68	__atg1_result; \
fe4610b3 RM	69	})
	70	#define __atomic_bool_bysize(pre, post, mem, ...) \
	71	({ \
11bf311e	72	int __atg2_result; \
fe4610b3	73	if (sizeof (*mem) == 1) \
11bf311e	74	__atg2_result = pre##_8_##post (mem, __VA_ARGS__); \
fe4610b3	75	else if (sizeof (*mem) == 2) \
11bf311e	76	__atg2_result = pre##_16_##post (mem, __VA_ARGS__); \
fe4610b3	77	else if (sizeof (*mem) == 4) \
11bf311e	78	__atg2_result = pre##_32_##post (mem, __VA_ARGS__); \
fe4610b3	79	else if (sizeof (*mem) == 8) \
11bf311e	80	__atg2_result = pre##_64_##post (mem, __VA_ARGS__); \
fe4610b3 RM	81	else \
fe4610b3 RM	82	abort (); \
11bf311e	83	__atg2_result; \
fe4610b3 RM	84	})
fe4610b3 RM	85
76a50749	86
d15851ec RM	87	/* Atomically store NEWVAL in MEM if MEM is equal to OLDVAL.
	88	Return the old MEM value. /
	89	#if !defined atomic_compare_and_exchange_val_acq \
	90	&& defined __arch_compare_and_exchange_val_32_acq
18627f61	91	# define atomic_compare_and_exchange_val_acq(mem, newval, oldval) \
a1b5424f UD	92	__atomic_val_bysize (__arch_compare_and_exchange_val,acq, \
a1b5424f UD	93	mem, newval, oldval)
76a50749 UD	94	#endif
	95
	96
3b1b533b JJ	97	#ifndef catomic_compare_and_exchange_val_acq
	98	# ifdef __arch_c_compare_and_exchange_val_32_acq
	99	# define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \
11bf311e UD	100	__atomic_val_bysize (__arch_c_compare_and_exchange_val,acq, \
11bf311e UD	101	mem, newval, oldval)
3b1b533b JJ	102	# else
3b1b533b JJ	103	# define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \
11bf311e	104	atomic_compare_and_exchange_val_acq (mem, newval, oldval)
3b1b533b	105	# endif
11bf311e UD	106	#endif
	107
	108
bea0ac1d JJ	109	#ifndef catomic_compare_and_exchange_val_rel
	110	# ifndef atomic_compare_and_exchange_val_rel
	111	# define catomic_compare_and_exchange_val_rel(mem, newval, oldval) \
	112	catomic_compare_and_exchange_val_acq (mem, newval, oldval)
	113	# else
	114	# define catomic_compare_and_exchange_val_rel(mem, newval, oldval) \
	115	atomic_compare_and_exchange_val_rel (mem, newval, oldval)
	116	# endif
18627f61 UD	117	#endif
	118
	119
bea0ac1d JJ	120	#ifndef atomic_compare_and_exchange_val_rel
bea0ac1d JJ	121	# define atomic_compare_and_exchange_val_rel(mem, newval, oldval) \
11bf311e UD	122	atomic_compare_and_exchange_val_acq (mem, newval, oldval)
	123	#endif
	124
	125
d15851ec RM	126	/* Atomically store NEWVAL in MEM if MEM is equal to OLDVAL.
d15851ec RM	127	Return zero if MEM was changed or non-zero if no exchange happened. /
18627f61 UD	128	#ifndef atomic_compare_and_exchange_bool_acq
18627f61 UD	129	# ifdef __arch_compare_and_exchange_bool_32_acq
18627f61	130	# define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
a1b5424f UD	131	__atomic_bool_bysize (__arch_compare_and_exchange_bool,acq, \
a1b5424f UD	132	mem, newval, oldval)
3b1b533b JJ	133	# else
3b1b533b JJ	134	# define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
d15851ec RM	135	({ /* Cannot use __oldval here, because macros later in this file might \
d15851ec RM	136	call this macro with __oldval argument. */ \
11bf311e UD	137	__typeof (oldval) __atg3_old = (oldval); \
	138	atomic_compare_and_exchange_val_acq (mem, newval, __atg3_old) \
	139	!= __atg3_old; \
	140	})
	141	# endif
	142	#endif
	143
	144
	145	#ifndef catomic_compare_and_exchange_bool_acq
	146	# ifdef __arch_c_compare_and_exchange_bool_32_acq
	147	# define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
	148	__atomic_bool_bysize (__arch_c_compare_and_exchange_bool,acq, \
	149	mem, newval, oldval)
3b1b533b JJ	150	# else
3b1b533b JJ	151	# define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
11bf311e UD	152	({ /* Cannot use __oldval here, because macros later in this file might \
	153	call this macro with __oldval argument. */ \
	154	__typeof (oldval) __atg4_old = (oldval); \
	155	catomic_compare_and_exchange_val_acq (mem, newval, __atg4_old) \
	156	!= __atg4_old; \
8099361e UD	157	})
	158	# endif
	159	#endif
	160
	161
bea0ac1d JJ	162	#ifndef catomic_compare_and_exchange_bool_rel
	163	# ifndef atomic_compare_and_exchange_bool_rel
	164	# define catomic_compare_and_exchange_bool_rel(mem, newval, oldval) \
	165	catomic_compare_and_exchange_bool_acq (mem, newval, oldval)
	166	# else
	167	# define catomic_compare_and_exchange_bool_rel(mem, newval, oldval) \
	168	atomic_compare_and_exchange_bool_rel (mem, newval, oldval)
	169	# endif
76a50749 UD	170	#endif
	171
	172
bea0ac1d JJ	173	#ifndef atomic_compare_and_exchange_bool_rel
	174	# define atomic_compare_and_exchange_bool_rel(mem, newval, oldval) \
	175	atomic_compare_and_exchange_bool_acq (mem, newval, oldval)
11bf311e UD	176	#endif
	177
	178
5132931e	179	/* Store NEWVALUE in MEM and return the old value. /
949ec764 UD	180	#ifndef atomic_exchange_acq
949ec764 UD	181	# define atomic_exchange_acq(mem, newvalue) \
11bf311e UD	182	({ __typeof (*(mem)) __atg5_oldval; \
	183	__typeof (mem) __atg5_memp = (mem); \
	184	__typeof (*(mem)) __atg5_value = (newvalue); \
5132931e UD	185	\
5132931e UD	186	do \
11bf311e UD	187	__atg5_oldval = *__atg5_memp; \
	188	while (__builtin_expect \
	189	(atomic_compare_and_exchange_bool_acq (__atg5_memp, __atg5_value, \
	190	__atg5_oldval), 0)); \
5132931e	191	\
11bf311e	192	__atg5_oldval; })
5132931e UD	193	#endif
5132931e UD	194
949ec764 UD	195	#ifndef atomic_exchange_rel
	196	# define atomic_exchange_rel(mem, newvalue) atomic_exchange_acq (mem, newvalue)
	197	#endif
	198
5132931e UD	199
5132931e UD	200	/* Add VALUE to MEM and return the old value of MEM. */
51a9ba86 MK	201	#ifndef atomic_exchange_and_add_acq
	202	# ifdef atomic_exchange_and_add
	203	# define atomic_exchange_and_add_acq(mem, value) \
	204	atomic_exchange_and_add (mem, value)
	205	# else
	206	# define atomic_exchange_and_add_acq(mem, value) \
11bf311e UD	207	({ __typeof (*(mem)) __atg6_oldval; \
	208	__typeof (mem) __atg6_memp = (mem); \
	209	__typeof (*(mem)) __atg6_value = (value); \
76a50749 UD	210	\
76a50749 UD	211	do \
11bf311e UD	212	__atg6_oldval = *__atg6_memp; \
	213	while (__builtin_expect \
	214	(atomic_compare_and_exchange_bool_acq (__atg6_memp, \
	215	__atg6_oldval \
	216	+ __atg6_value, \
	217	__atg6_oldval), 0)); \
76a50749	218	\
11bf311e	219	__atg6_oldval; })
51a9ba86	220	# endif
76a50749 UD	221	#endif
76a50749 UD	222
51a9ba86 MK	223	#ifndef atomic_exchange_and_add_rel
	224	# define atomic_exchange_and_add_rel(mem, value) \
	225	atomic_exchange_and_add_acq(mem, value)
	226	#endif
	227
	228	#ifndef atomic_exchange_and_add
	229	# define atomic_exchange_and_add(mem, value) \
	230	atomic_exchange_and_add_acq(mem, value)
	231	#endif
76a50749	232
11bf311e UD	233	#ifndef catomic_exchange_and_add
	234	# define catomic_exchange_and_add(mem, value) \
	235	({ __typeof (*(mem)) __atg7_oldv; \
	236	__typeof (mem) __atg7_memp = (mem); \
	237	__typeof (*(mem)) __atg7_value = (value); \
	238	\
	239	do \
	240	__atg7_oldv = *__atg7_memp; \
	241	while (__builtin_expect \
	242	(catomic_compare_and_exchange_bool_acq (__atg7_memp, \
	243	__atg7_oldv \
	244	+ __atg7_value, \
	245	__atg7_oldv), 0)); \
	246	\
	247	__atg7_oldv; })
	248	#endif
	249
d6c7294e UD	250
	251	#ifndef atomic_max
	252	# define atomic_max(mem, value) \
	253	do { \
11bf311e UD	254	__typeof (*(mem)) __atg8_oldval; \
	255	__typeof (mem) __atg8_memp = (mem); \
	256	__typeof (*(mem)) __atg8_value = (value); \
	257	do { \
	258	__atg8_oldval = *__atg8_memp; \
	259	if (__atg8_oldval >= __atg8_value) \
	260	break; \
	261	} while (__builtin_expect \
	262	(atomic_compare_and_exchange_bool_acq (__atg8_memp, __atg8_value,\
	263	__atg8_oldval), 0)); \
	264	} while (0)
	265	#endif
	266
	267
	268	#ifndef catomic_max
	269	# define catomic_max(mem, value) \
	270	do { \
	271	__typeof (*(mem)) __atg9_oldv; \
	272	__typeof (mem) __atg9_memp = (mem); \
	273	__typeof (*(mem)) __atg9_value = (value); \
8099361e	274	do { \
11bf311e UD	275	__atg9_oldv = *__atg9_memp; \
11bf311e UD	276	if (__atg9_oldv >= __atg9_value) \
8099361e	277	break; \
11bf311e UD	278	} while (__builtin_expect \
	279	(catomic_compare_and_exchange_bool_acq (__atg9_memp, \
	280	__atg9_value, \
	281	__atg9_oldv), 0)); \
8099361e UD	282	} while (0)
	283	#endif
	284
11bf311e	285
d6c7294e UD	286	#ifndef atomic_min
	287	# define atomic_min(mem, value) \
	288	do { \
11bf311e UD	289	__typeof (*(mem)) __atg10_oldval; \
	290	__typeof (mem) __atg10_memp = (mem); \
	291	__typeof (*(mem)) __atg10_value = (value); \
d6c7294e	292	do { \
11bf311e UD	293	__atg10_oldval = *__atg10_memp; \
11bf311e UD	294	if (__atg10_oldval <= __atg10_value) \
d6c7294e	295	break; \
11bf311e UD	296	} while (__builtin_expect \
	297	(atomic_compare_and_exchange_bool_acq (__atg10_memp, \
	298	__atg10_value, \
	299	__atg10_oldval), 0)); \
d6c7294e UD	300	} while (0)
	301	#endif
	302
11bf311e	303
76a50749	304	#ifndef atomic_add
fe4610b3	305	# define atomic_add(mem, value) (void) atomic_exchange_and_add ((mem), (value))
76a50749 UD	306	#endif
	307
	308
11bf311e UD	309	#ifndef catomic_add
	310	# define catomic_add(mem, value) \
	311	(void) catomic_exchange_and_add ((mem), (value))
	312	#endif
	313
	314
76a50749	315	#ifndef atomic_increment
fe4610b3	316	# define atomic_increment(mem) atomic_add ((mem), 1)
76a50749 UD	317	#endif
	318
	319
11bf311e UD	320	#ifndef catomic_increment
	321	# define catomic_increment(mem) catomic_add ((mem), 1)
	322	#endif
	323
	324
f377d022 UD	325	#ifndef atomic_increment_val
	326	# define atomic_increment_val(mem) (atomic_exchange_and_add ((mem), 1) + 1)
	327	#endif
	328
	329
11bf311e UD	330	#ifndef catomic_increment_val
	331	# define catomic_increment_val(mem) (catomic_exchange_and_add ((mem), 1) + 1)
	332	#endif
	333
	334
fe4610b3	335	/* Add one to MEM and return true iff it's now zero. /
ec609a8e UD	336	#ifndef atomic_increment_and_test
ec609a8e UD	337	# define atomic_increment_and_test(mem) \
fe4610b3	338	(atomic_exchange_and_add ((mem), 1) + 1 == 0)
ec609a8e UD	339	#endif
	340
	341
76a50749	342	#ifndef atomic_decrement
fe4610b3	343	# define atomic_decrement(mem) atomic_add ((mem), -1)
76a50749 UD	344	#endif
	345
	346
11bf311e UD	347	#ifndef catomic_decrement
	348	# define catomic_decrement(mem) catomic_add ((mem), -1)
	349	#endif
	350
	351
f377d022 UD	352	#ifndef atomic_decrement_val
	353	# define atomic_decrement_val(mem) (atomic_exchange_and_add ((mem), -1) - 1)
	354	#endif
	355
	356
11bf311e UD	357	#ifndef catomic_decrement_val
	358	# define catomic_decrement_val(mem) (catomic_exchange_and_add ((mem), -1) - 1)
	359	#endif
	360
	361
d15851ec	362	/* Subtract 1 from MEM and return true iff it's now zero. /
ec609a8e UD	363	#ifndef atomic_decrement_and_test
ec609a8e UD	364	# define atomic_decrement_and_test(mem) \
fe4610b3	365	(atomic_exchange_and_add ((mem), -1) == 1)
ec609a8e UD	366	#endif
	367
	368
5132931e	369	/* Decrement MEM if it is > 0, and return the old value. /
2ff16e04	370	#ifndef atomic_decrement_if_positive
972209dd	371	# define atomic_decrement_if_positive(mem) \
11bf311e UD	372	({ __typeof (*(mem)) __atg11_oldval; \
11bf311e UD	373	__typeof (mem) __atg11_memp = (mem); \
972209dd	374	\
5132931e UD	375	do \
5132931e UD	376	{ \
11bf311e	377	__atg11_oldval = *__atg11_memp; \
a1ffb40e	378	if (__glibc_unlikely (__atg11_oldval <= 0)) \
5132931e	379	break; \
5132931e	380	} \
11bf311e UD	381	while (__builtin_expect \
	382	(atomic_compare_and_exchange_bool_acq (__atg11_memp, \
	383	__atg11_oldval - 1, \
	384	__atg11_oldval), 0)); \
	385	__atg11_oldval; })
5132931e UD	386	#endif
	387
	388
5cd09cd6	389	#ifndef atomic_add_negative
859e708f	390	# define atomic_add_negative(mem, value) \
11bf311e UD	391	({ __typeof (value) __atg12_value = (value); \
11bf311e UD	392	atomic_exchange_and_add (mem, __atg12_value) < -__atg12_value; })
ec609a8e UD	393	#endif
	394
	395
	396	#ifndef atomic_add_zero
859e708f	397	# define atomic_add_zero(mem, value) \
11bf311e UD	398	({ __typeof (value) __atg13_value = (value); \
11bf311e UD	399	atomic_exchange_and_add (mem, __atg13_value) == -__atg13_value; })
ec609a8e UD	400	#endif
	401
	402
0289bef9 UD	403	#ifndef atomic_bit_set
0289bef9 UD	404	# define atomic_bit_set(mem, bit) \
a1b5424f	405	(void) atomic_bit_test_set(mem, bit)
ec609a8e UD	406	#endif
	407
	408
	409	#ifndef atomic_bit_test_set
	410	# define atomic_bit_test_set(mem, bit) \
11bf311e UD	411	({ __typeof (*(mem)) __atg14_old; \
	412	__typeof (mem) __atg14_memp = (mem); \
	413	__typeof ((mem)) __atg14_mask = ((__typeof ((mem))) 1 << (bit)); \
ec609a8e UD	414	\
ec609a8e UD	415	do \
11bf311e UD	416	__atg14_old = (*__atg14_memp); \
	417	while (__builtin_expect \
	418	(atomic_compare_and_exchange_bool_acq (__atg14_memp, \
	419	__atg14_old \| __atg14_mask,\
	420	__atg14_old), 0)); \
76a50749	421	\
11bf311e	422	__atg14_old & __atg14_mask; })
76a50749 UD	423	#endif
76a50749 UD	424
11bf311e	425	/* Atomically mem &= mask. /
f1740bc4 UD	426	#ifndef atomic_and
f1740bc4 UD	427	# define atomic_and(mem, mask) \
11bf311e UD	428	do { \
	429	__typeof (*(mem)) __atg15_old; \
	430	__typeof (mem) __atg15_memp = (mem); \
	431	__typeof (*(mem)) __atg15_mask = (mask); \
	432	\
	433	do \
	434	__atg15_old = (*__atg15_memp); \
	435	while (__builtin_expect \
	436	(atomic_compare_and_exchange_bool_acq (__atg15_memp, \
	437	__atg15_old & __atg15_mask, \
	438	__atg15_old), 0)); \
	439	} while (0)
	440	#endif
	441
6c03cd11 UD	442	#ifndef catomic_and
	443	# define catomic_and(mem, mask) \
	444	do { \
	445	__typeof (*(mem)) __atg20_old; \
	446	__typeof (mem) __atg20_memp = (mem); \
	447	__typeof (*(mem)) __atg20_mask = (mask); \
	448	\
	449	do \
	450	__atg20_old = (*__atg20_memp); \
	451	while (__builtin_expect \
	452	(catomic_compare_and_exchange_bool_acq (__atg20_memp, \
	453	__atg20_old & __atg20_mask,\
	454	__atg20_old), 0)); \
	455	} while (0)
	456	#endif
	457
11bf311e UD	458	/* Atomically mem &= mask and return the old value of mem. */
	459	#ifndef atomic_and_val
	460	# define atomic_and_val(mem, mask) \
	461	({ __typeof (*(mem)) __atg16_old; \
	462	__typeof (mem) __atg16_memp = (mem); \
	463	__typeof (*(mem)) __atg16_mask = (mask); \
f1740bc4 UD	464	\
f1740bc4 UD	465	do \
11bf311e UD	466	__atg16_old = (*__atg16_memp); \
	467	while (__builtin_expect \
	468	(atomic_compare_and_exchange_bool_acq (__atg16_memp, \
	469	__atg16_old & __atg16_mask,\
	470	__atg16_old), 0)); \
f1740bc4	471	\
11bf311e	472	__atg16_old; })
f1740bc4 UD	473	#endif
	474
	475	/* Atomically mem \|= mask and return the old value of mem. */
	476	#ifndef atomic_or
	477	# define atomic_or(mem, mask) \
11bf311e UD	478	do { \
	479	__typeof (*(mem)) __atg17_old; \
	480	__typeof (mem) __atg17_memp = (mem); \
	481	__typeof (*(mem)) __atg17_mask = (mask); \
	482	\
	483	do \
	484	__atg17_old = (*__atg17_memp); \
	485	while (__builtin_expect \
	486	(atomic_compare_and_exchange_bool_acq (__atg17_memp, \
	487	__atg17_old \| __atg17_mask, \
	488	__atg17_old), 0)); \
	489	} while (0)
	490	#endif
	491
	492	#ifndef catomic_or
	493	# define catomic_or(mem, mask) \
	494	do { \
	495	__typeof (*(mem)) __atg18_old; \
	496	__typeof (mem) __atg18_memp = (mem); \
	497	__typeof (*(mem)) __atg18_mask = (mask); \
	498	\
	499	do \
	500	__atg18_old = (*__atg18_memp); \
	501	while (__builtin_expect \
	502	(catomic_compare_and_exchange_bool_acq (__atg18_memp, \
	503	__atg18_old \| __atg18_mask,\
	504	__atg18_old), 0)); \
	505	} while (0)
	506	#endif
	507
	508	/* Atomically mem \|= mask and return the old value of mem. */
	509	#ifndef atomic_or_val
	510	# define atomic_or_val(mem, mask) \
	511	({ __typeof (*(mem)) __atg19_old; \
	512	__typeof (mem) __atg19_memp = (mem); \
	513	__typeof (*(mem)) __atg19_mask = (mask); \
f1740bc4 UD	514	\
f1740bc4 UD	515	do \
11bf311e UD	516	__atg19_old = (*__atg19_memp); \
	517	while (__builtin_expect \
	518	(atomic_compare_and_exchange_bool_acq (__atg19_memp, \
	519	__atg19_old \| __atg19_mask,\
	520	__atg19_old), 0)); \
f1740bc4	521	\
11bf311e	522	__atg19_old; })
f1740bc4	523	#endif
76a50749 UD	524
76a50749 UD	525	#ifndef atomic_full_barrier
733f25e6	526	# define atomic_full_barrier() __asm ("" ::: "memory")
76a50749 UD	527	#endif
	528
	529
	530	#ifndef atomic_read_barrier
a1b5424f	531	# define atomic_read_barrier() atomic_full_barrier ()
76a50749 UD	532	#endif
	533
	534
	535	#ifndef atomic_write_barrier
a1b5424f	536	# define atomic_write_barrier() atomic_full_barrier ()
76a50749 UD	537	#endif
76a50749 UD	538
f377d022	539
fc242bef UD	540	#ifndef atomic_forced_read
	541	# define atomic_forced_read(x) \
	542	({ __typeof (x) __x; __asm ("" : "=r" (__x) : "0" (x)); __x; })
	543	#endif
	544
	545
f377d022 UD	546	#ifndef atomic_delay
	547	# define atomic_delay() do { /* nothing */ } while (0)
	548	#endif
	549
76a50749	550	#endif /* atomic.h */