Clean up BUSY_WAIT_NOP and atomic_delay.

This patch combines BUSY_WAIT_NOP and atomic_delay into a new
atomic_spin_nop function and adjusts all clients.  The new function is
put into atomic.h because what is best done in a spin loop is
architecture-specific, and atomics must be used for spinning.  The
function name is meant to tell users that this has no effect on
synchronization semantics but is a performance aid for spinning.

diff --git a/ChangeLog b/ChangeLog
index 41176b5fc81f08ee6a6d21e4be69f60ebab03f33..0e22a702e9ef1b52a043e1976dd678ef22dbfad5 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,26 @@
+2015-06-30  Torvald Riegel  <triegel@redhat.com>
+
+       * sysdeps/unix/sysv/linux/i386/lowlevellock.h (BUSY_WAIT_NOP): Remove.
+       * sysdeps/unix/sysv/linux/x86_64/lowlevellock.h (BUSY_WAIT_NOP):
+       Likewise.
+       * sysdeps/i386/i486/bits/atomic.h (atomic_delay): Rename to
+       atomic_spin_nop.
+       * sysdeps/x86_64/bits/atomic.h: Likewise.
+       * sysdeps/unix/sysv/linux/sparc/lowlevellock.h (BUSY_WAIT_NOP): Rename
+       to atomic_spin_nop and move ...
+       * sysdeps/sparc/sparc32/sparcv9/bits/atomic.h (atomic_spin_nop):
+       ... here and ...
+       * sysdeps/sparc/sparc64/bits/atomic.h: ... here.
+       * nptl/pthread_mutex_lock.c (__pthread_mutex_lock): Use
+       atomic_spin_nop instead of BUSY_WAIT_NOP.
+       * nptl/pthread_mutex_timedlock.c (__pthread_mutex_timedlock):
+       Likewise.
+       * sysdeps/nacl/lll_timedwait_tid.c (__lll_timedwait_tid): Likewise.
+       * sysdeps/nacl/lowlevellock.h (BUSY_WAIT_NOP): Remove.
+       (lll_wait_tid): Use atomic_spin_nop instead of BUSY_WAIT_NOP.
+       * nscd/nscd-client.h (__nscd_acquire_maplock): Use atomic_spin_nop
+       instead of atomic_delay.
+
 2015-06-29  Joseph Myers  <joseph@codesourcery.com>
 
        [BZ #18613]

diff --git a/include/atomic.h b/include/atomic.h
index 7fd70c4c81609a3115f223d4868893a9406b2da3..221bea0ad354a3d2d373ea421519d533f93e1cf4 100644 (file)
--- a/include/atomic.h
+++ b/include/atomic.h
@@ -754,9 +754,10 @@ void __atomic_link_error (void);
 
 #endif /* !USE_ATOMIC_COMPILER_BUILTINS  */
 
-
-#ifndef atomic_delay
-# define atomic_delay() do { /* nothing */ } while (0)
+/* This operation does not affect synchronization semantics but can be used
+   in the body of a spin loop to potentially improve its efficiency.  */
+#ifndef atomic_spin_nop
+# define atomic_spin_nop() do { /* nothing */ } while (0)
 #endif
 
 #endif /* atomic.h */

diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c
index 96075129b8df840f76127e614871efb3c2d5795f..9a3b46624d4cf5b2507002ef5cf488a2992230ce 100644 (file)
--- a/nptl/pthread_mutex_lock.c
+++ b/nptl/pthread_mutex_lock.c
@@ -23,6 +23,7 @@
 #include <sys/param.h>
 #include <not-cancel.h>
 #include "pthreadP.h"
+#include <atomic.h>
 #include <lowlevellock.h>
 #include <stap-probe.h>
 
@@ -135,10 +136,7 @@ __pthread_mutex_lock (mutex)
                  LLL_MUTEX_LOCK (mutex);
                  break;
                }
-
-#ifdef BUSY_WAIT_NOP
-             BUSY_WAIT_NOP;
-#endif
+             atomic_spin_nop ();
            }
          while (LLL_MUTEX_TRYLOCK (mutex) != 0);
 

diff --git a/nptl/pthread_mutex_timedlock.c b/nptl/pthread_mutex_timedlock.c
index 109a46a84d0c8d901281425dcb7f32f29e5467cf..f0fb03e90bac3773c3cd612d57c2a5fa9ababcda 100644 (file)
--- a/nptl/pthread_mutex_timedlock.c
+++ b/nptl/pthread_mutex_timedlock.c
@@ -22,6 +22,7 @@
 #include <sys/param.h>
 #include <sys/time.h>
 #include "pthreadP.h"
+#include <atomic.h>
 #include <lowlevellock.h>
 #include <not-cancel.h>
 
@@ -125,10 +126,7 @@ pthread_mutex_timedlock (mutex, abstime)
                                          PTHREAD_MUTEX_PSHARED (mutex));
                  break;
                }
-
-#ifdef BUSY_WAIT_NOP
-             BUSY_WAIT_NOP;
-#endif
+             atomic_spin_nop ();
            }
          while (lll_trylock (mutex->__data.__lock) != 0);
 

diff --git a/nscd/nscd-client.h b/nscd/nscd-client.h
index 43a8c6101496733ca8546031d0bb3991c79afbda..740e2f9e342f42cae31a41f4090619c1484b700a 100644 (file)
--- a/nscd/nscd-client.h
+++ b/nscd/nscd-client.h
@@ -378,7 +378,7 @@ __nscd_acquire_maplock (volatile struct locked_map_ptr *mapptr)
       if (__glibc_unlikely (++cnt > 5))
        return false;
 
-      atomic_delay ();
+      atomic_spin_nop ();
     }
 
   return true;

diff --git a/sysdeps/i386/i486/bits/atomic.h b/sysdeps/i386/i486/bits/atomic.h
index 59165bec94389df238311f4a8271e96f519805a5..59f3d34871a16cc9e3fe04db1b00c5884dc851ba 100644 (file)
--- a/sysdeps/i386/i486/bits/atomic.h
+++ b/sysdeps/i386/i486/bits/atomic.h
@@ -479,7 +479,7 @@ typedef uintmax_t uatomic_max_t;
      __result; })
 
 
-#define atomic_delay() asm ("rep; nop")
+#define atomic_spin_nop() asm ("rep; nop")
 
 
 #define __arch_and_body(lock, mem, mask) \

diff --git a/sysdeps/nacl/lll_timedwait_tid.c b/sysdeps/nacl/lll_timedwait_tid.c
index ecaf0b113a9607c83b7da4090194e62d677079a7..ef544cf84f16a43bad8e9d82ab2cc7de0effb8ad 100644 (file)
--- a/sysdeps/nacl/lll_timedwait_tid.c
+++ b/sysdeps/nacl/lll_timedwait_tid.c
@@ -40,7 +40,7 @@ __lll_timedwait_tid (int *tidp, const struct timespec *abstime)
           finish quick enough that the timeout doesn't matter.  If any
           thread ever stays in this state for long, there is something
           catastrophically wrong.  */
-       BUSY_WAIT_NOP;
+       atomic_spin_nop ();
       else
        {
          assert (tid > 0);

diff --git a/sysdeps/nacl/lowlevellock.h b/sysdeps/nacl/lowlevellock.h
index 0b85d8d3173694016ff43c4c2a8edba7500a38b8..3634f1959a09a7850e8509d008bcb21166b84a89 100644 (file)
--- a/sysdeps/nacl/lowlevellock.h
+++ b/sysdeps/nacl/lowlevellock.h
@@ -21,10 +21,6 @@
 /* Everything except the exit handling is the same as the generic code.  */
 # include <sysdeps/nptl/lowlevellock.h>
 
-# ifndef BUSY_WAIT_NOP
-#  define BUSY_WAIT_NOP                __sync_synchronize ()
-# endif
-
 /* See exit-thread.h for details.  */
 # define NACL_EXITING_TID      1
 
@@ -36,7 +32,7 @@
     while ((__tid = atomic_load_relaxed (__tidp)) != 0) \
       {                                                        \
        if (__tid == NACL_EXITING_TID)                  \
-         BUSY_WAIT_NOP;                                \
+         atomic_spin_nop ();                           \
        else                                            \
          lll_futex_wait (__tidp, __tid, LLL_PRIVATE);  \
       }                                                        \

diff --git a/sysdeps/sparc/sparc32/sparcv9/bits/atomic.h b/sysdeps/sparc/sparc32/sparcv9/bits/atomic.h
index 317be62ccbb5fcb436bf510a35dbdff1e3b6dabf..2122afbb0924076a29618c168ddbce2a6ddb3d58 100644 (file)
--- a/sysdeps/sparc/sparc32/sparcv9/bits/atomic.h
+++ b/sysdeps/sparc/sparc32/sparcv9/bits/atomic.h
@@ -100,3 +100,6 @@ typedef uintmax_t uatomic_max_t;
   __asm __volatile ("membar #LoadLoad | #LoadStore" : : : "memory")
 #define atomic_write_barrier() \
   __asm __volatile ("membar #LoadStore | #StoreStore" : : : "memory")
+
+extern void __cpu_relax (void);
+#define atomic_spin_nop () __cpu_relax ()

diff --git a/sysdeps/sparc/sparc64/bits/atomic.h b/sysdeps/sparc/sparc64/bits/atomic.h
index 35804a8e14251521525c9bc7af0059a4b3992127..48b7fd621676f424cc914aa9d8b6d7ce735a00b8 100644 (file)
--- a/sysdeps/sparc/sparc64/bits/atomic.h
+++ b/sysdeps/sparc/sparc64/bits/atomic.h
@@ -121,3 +121,6 @@ typedef uintmax_t uatomic_max_t;
   __asm __volatile ("membar #LoadLoad | #LoadStore" : : : "memory")
 #define atomic_write_barrier() \
   __asm __volatile ("membar #LoadStore | #StoreStore" : : : "memory")
+
+extern void __cpu_relax (void);
+#define atomic_spin_nop () __cpu_relax ()

diff --git a/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/sysdeps/unix/sysv/linux/i386/lowlevellock.h
index f57afc6e2a90f47e98bc20b187812b66557d4386..58f5638e37ce9d8e6bab8e39c7211fa564dbf1f4 100644 (file)
--- a/sysdeps/unix/sysv/linux/i386/lowlevellock.h
+++ b/sysdeps/unix/sysv/linux/i386/lowlevellock.h
@@ -58,10 +58,6 @@
 #define LLL_LOCK_INITIALIZER_WAITERS   (2)
 
 
-/* Delay in spinlock loop.  */
-#define BUSY_WAIT_NOP  asm ("rep; nop")
-
-
 /* NB: in the lll_trylock macro we simply return the value in %eax
    after the cmpxchg instruction.  In case the operation succeded this
    value is zero.  In case the operation failed, the cmpxchg instruction

diff --git a/sysdeps/unix/sysv/linux/sparc/lowlevellock.h b/sysdeps/unix/sysv/linux/sparc/lowlevellock.h
index 9aefd9eb590d560143ede7fca7a3ce73a6ff4054..7608c01d17f7e30ea91b02ccb148c2b0f112668b 100644 (file)
--- a/sysdeps/unix/sysv/linux/sparc/lowlevellock.h
+++ b/sysdeps/unix/sysv/linux/sparc/lowlevellock.h
@@ -25,12 +25,6 @@
 #include <atomic.h>
 #include <kernel-features.h>
 
-#ifndef __sparc32_atomic_do_lock
-/* Delay in spinlock loop.  */
-extern void __cpu_relax (void);
-#define BUSY_WAIT_NOP  __cpu_relax ()
-#endif
-
 #include <lowlevellock-futex.h>
 
 static inline int

diff --git a/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
index 573b48c4fece143ba7d28eb23043fd23f5443648..de525cd4c76595624b5095ef0758bc6d77df9023 100644 (file)
--- a/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
+++ b/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
@@ -57,9 +57,6 @@
 #define LLL_LOCK_INITIALIZER_LOCKED    (1)
 #define LLL_LOCK_INITIALIZER_WAITERS   (2)
 
-/* Delay in spinlock loop.  */
-#define BUSY_WAIT_NOP    asm ("rep; nop")
-
 
 /* NB: in the lll_trylock macro we simply return the value in %eax
    after the cmpxchg instruction.  In case the operation succeded this

diff --git a/sysdeps/x86_64/bits/atomic.h b/sysdeps/x86_64/bits/atomic.h
index 203d92c20d05c47faf3a82ef9e7732723e4c39c3..337b334db1cf1970a59379b1ca69cf4e1ebf9d39 100644 (file)
--- a/sysdeps/x86_64/bits/atomic.h
+++ b/sysdeps/x86_64/bits/atomic.h
@@ -410,7 +410,7 @@ typedef uintmax_t uatomic_max_t;
      __result; })
 
 
-#define atomic_delay() asm ("rep; nop")
+#define atomic_spin_nop() asm ("rep; nop")
 
 
 #define __arch_and_body(lock, mem, mask) \