nptl: Replace non cancellable pause/nanosleep with futex

To help y2038 work avoid duplicate all the logic of nanosleep on
non cancellable version, the patch replace it with a new futex
operation, lll_timedwait.  The changes are:

  - Add a expected value for __lll_clocklock_wait, so it can be used
    to wait for generic values.

  - Remove its internal atomic operation and move the logic to
    __lll_clocklock.  It makes __lll_clocklock_wait even more generic
    and __lll_clocklock slight faster on fast-path (since it won't
    require a function call anymore).

  - Add lll_timedwait, which uses __lll_clocklock_wait, to replace both
    __pause_nocancel and __nanosleep_nocancel.

It also allows remove the sparc32 __lll_clocklock_wait implementation
(since it is similar to the generic one).

Checked on x86_64-linux-gnu, sparcv9-linux-gnu, and i686-linux-gnu.

Reviewed-by: Carlos O'Donell <carlos@redhat.com>

diff --git a/nptl/lll_timedlock_wait.c b/nptl/lll_timedlock_wait.c
index cd3cc3d3715319487d6664cbbd8a1df529327ea8..952b042555bc979676d3e61934387a0b5303424a 100644 (file)
--- a/nptl/lll_timedlock_wait.c
+++ b/nptl/lll_timedlock_wait.c
@@ -25,39 +25,38 @@
 
 
 int
-__lll_clocklock_wait (int *futex, clockid_t clockid,
+__lll_clocklock_wait (int *futex, int val, clockid_t clockid,
                      const struct timespec *abstime, int private)
 {
-  /* Reject invalid timeouts.  */
-  if (! valid_nanoseconds (abstime->tv_nsec))
-    return EINVAL;
+  struct timespec ts, *tsp = NULL;
 
-  /* Try locking.  */
-  while (atomic_exchange_acq (futex, 2) != 0)
+  if (abstime != NULL)
     {
-      struct timespec ts;
+      /* Reject invalid timeouts.  */
+      if (! valid_nanoseconds (abstime->tv_nsec))
+        return EINVAL;
 
-      /* Get the current time. This can only fail if clockid is not
-         valid.  */
+      /* Get the current time. This can only fail if clockid is not valid.  */
       if (__glibc_unlikely (__clock_gettime (clockid, &ts) != 0))
         return EINVAL;
 
       /* Compute relative timeout.  */
-      struct timespec rt;
-      rt.tv_sec = abstime->tv_sec - ts.tv_sec;
-      rt.tv_nsec = abstime->tv_nsec - ts.tv_nsec;
-      if (rt.tv_nsec < 0)
+      ts.tv_sec = abstime->tv_sec - ts.tv_sec;
+      ts.tv_nsec = abstime->tv_nsec - ts.tv_nsec;
+      if (ts.tv_nsec < 0)
         {
-          rt.tv_nsec += 1000000000;
-          --rt.tv_sec;
+         ts.tv_nsec += 1000000000;
+         --ts.tv_sec;
         }
 
-      if (rt.tv_sec < 0)
+      if (ts.tv_sec < 0)
         return ETIMEDOUT;
 
-      /* If *futex == 2, wait until woken or timeout.  */
-      lll_futex_timed_wait (futex, 2, &rt, private);
+      tsp = &ts;
     }
 
+  /* If *futex == val, wait until woken or timeout.  */
+  lll_futex_timed_wait (futex, val, tsp, private);
+
   return 0;
 }

diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c
index 0ab890d8158a9eb58320e7784756ed61e001ba3b..ace436d5a6d712b47ad9001685f83543b1b946fd 100644 (file)
--- a/nptl/pthread_mutex_lock.c
+++ b/nptl/pthread_mutex_lock.c
@@ -434,7 +434,8 @@ __pthread_mutex_lock_full (pthread_mutex_t *mutex)
 
                /* Delay the thread indefinitely.  */
                while (1)
-                 __pause_nocancel ();
+                 lll_timedwait (&(int){0}, 0, 0 /* ignored */, NULL,
+                                private);
              }
 
            oldval = mutex->__data.__lock;

diff --git a/nptl/pthread_mutex_timedlock.c b/nptl/pthread_mutex_timedlock.c
index d6f0d9e31a83b46829433e64cc57643c3a5ff6e9..76b93bddc6db98b0f96a78d6474ee657e16d78d8 100644 (file)
--- a/nptl/pthread_mutex_timedlock.c
+++ b/nptl/pthread_mutex_timedlock.c
@@ -401,22 +401,10 @@ __pthread_mutex_clocklock_common (pthread_mutex_t *mutex,
 
                    /* Delay the thread until the timeout is reached.
                       Then return ETIMEDOUT.  */
-                   struct timespec reltime;
-                   struct timespec now;
-
-                   INTERNAL_SYSCALL (clock_gettime, __err, 2, clockid,
-                                     &now);
-                   reltime.tv_sec = abstime->tv_sec - now.tv_sec;
-                   reltime.tv_nsec = abstime->tv_nsec - now.tv_nsec;
-                   if (reltime.tv_nsec < 0)
-                     {
-                       reltime.tv_nsec += 1000000000;
-                       --reltime.tv_sec;
-                     }
-                   if (reltime.tv_sec >= 0)
-                     while (__nanosleep_nocancel (&reltime, &reltime) != 0)
-                       continue;
-
+                   do
+                     e = lll_timedwait (&(int){0}, 0, clockid, abstime,
+                                        private);
+                   while (e != ETIMEDOUT);
                    return ETIMEDOUT;
                  }
 

diff --git a/sysdeps/nptl/lowlevellock.h b/sysdeps/nptl/lowlevellock.h
index e3f006537a387f26ba47d25dca909cd7024cbb7b..92298f3b7edfad2f18e4937fd30dffc08c616791 100644 (file)
--- a/sysdeps/nptl/lowlevellock.h
+++ b/sysdeps/nptl/lowlevellock.h
@@ -21,6 +21,7 @@
 
 #include <atomic.h>
 #include <lowlevellock-futex.h>
+#include <time.h>
 
 /* Low-level locks use a combination of atomic operations (to acquire and
    release lock ownership) and futex operations (to block until the state
@@ -121,10 +122,12 @@ extern void __lll_lock_wait (int *futex, int private) attribute_hidden;
 #define lll_cond_lock(futex, private) __lll_cond_lock (&(futex), private)
 
 
-extern int __lll_clocklock_wait (int *futex, clockid_t,
+extern int __lll_clocklock_wait (int *futex, int val, clockid_t,
                                 const struct timespec *,
                                 int private) attribute_hidden;
 
+#define lll_timedwait(futex, val, clockid, abstime, private)           \
+  __lll_clocklock_wait (futex, val, clockid, abstime, private)
 
 /* As __lll_lock, but with an absolute timeout measured against the clock
    specified in CLOCKID.  If the timeout occurs then return ETIMEDOUT. If
@@ -136,7 +139,15 @@ extern int __lll_clocklock_wait (int *futex, clockid_t,
                                                                 \
     if (__glibc_unlikely                                        \
         (atomic_compare_and_exchange_bool_acq (__futex, 1, 0))) \
-      __val = __lll_clocklock_wait (__futex, clockid, abstime, private); \
+      {                                                                \
+       while (atomic_exchange_acq (futex, 2) != 0)             \
+         {                                                     \
+           __val = __lll_clocklock_wait (__futex, 2, clockid,  \
+                                         abstime, private);    \
+           if (__val == EINVAL || __val == ETIMEDOUT)          \
+             break;                                            \
+         }                                                     \
+      }                                                                \
     __val;                                                      \
   })
 #define lll_clocklock(futex, clockid, abstime, private)         \

diff --git a/sysdeps/sparc/sparc32/lll_timedlock_wait.c b/sysdeps/sparc/sparc32/lll_timedlock_wait.c
deleted file mode 100644 (file)
index bd639a7..0000000
--- a/sysdeps/sparc/sparc32/lll_timedlock_wait.c
+++ /dev/null
@@ -1 +0,0 @@
-/* __lll_clocklock_wait is in lowlevellock.c.  */

diff --git a/sysdeps/sparc/sparc32/lowlevellock.c b/sysdeps/sparc/sparc32/lowlevellock.c
index 074ecf0636f47f383913112719ff6018221e3912..1a0ab4d9f23d43c10b8455606ea254b9475a2103 100644 (file)
--- a/sysdeps/sparc/sparc32/lowlevellock.c
+++ b/sysdeps/sparc/sparc32/lowlevellock.c
@@ -50,46 +50,4 @@ __lll_lock_wait (int *futex, int private)
     }
   while (atomic_compare_and_exchange_val_24_acq (futex, 2, 0) != 0);
 }
-
-
-int
-__lll_clocklock_wait (int *futex, clockid_t clockid,
-                      const struct timespec *abstime, int private)
-{
-  /* Reject invalid timeouts.  */
-  if (! valid_nanoseconds (abstime->tv_nsec))
-    return EINVAL;
-
-  do
-    {
-      struct timespec ts;
-      struct timespec rt;
-
-      /* Get the current time. This can only fail if clockid is not
-         valid. */
-      if (__glibc_unlikely (__clock_gettime (clockid, &ts) != 0))
-        return EINVAL;
-
-      /* Compute relative timeout.  */
-      rt.tv_sec = abstime->tv_sec - ts.tv_sec;
-      rt.tv_nsec = abstime->tv_nsec - ts.tv_nsec;
-      if (rt.tv_nsec < 0)
-       {
-         rt.tv_nsec += 1000000000;
-         --rt.tv_sec;
-       }
-
-      /* Already timed out?  */
-      if (rt.tv_sec < 0)
-       return ETIMEDOUT;
-
-      /* Wait.  */
-      int oldval = atomic_compare_and_exchange_val_24_acq (futex, 2, 1);
-      if (oldval != 0)
-       lll_futex_timed_wait (futex, 2, &rt, private);
-    }
-  while (atomic_compare_and_exchange_val_24_acq (futex, 2, 0) != 0);
-
-  return 0;
-}
 #endif

diff --git a/sysdeps/unix/sysv/linux/sparc/lowlevellock.h b/sysdeps/unix/sysv/linux/sparc/lowlevellock.h
index 34441f8ff5618432b8e87c6880af240991e17712..6c005e8ffa85720d8333c0f1e007a141d8e407a9 100644 (file)
--- a/sysdeps/unix/sysv/linux/sparc/lowlevellock.h
+++ b/sysdeps/unix/sysv/linux/sparc/lowlevellock.h
@@ -24,6 +24,7 @@
 #include <bits/pthreadtypes.h>
 #include <atomic.h>
 #include <kernel-features.h>
+#include <errno.h>
 
 #include <lowlevellock-futex.h>
 
@@ -75,9 +76,13 @@ __lll_cond_lock (int *futex, int private)
 #define lll_cond_lock(futex, private) __lll_cond_lock (&(futex), private)
 
 
-extern int __lll_clocklock_wait (int *futex, clockid_t, const struct timespec *,
+extern int __lll_clocklock_wait (int *futex, clockid_t, int val,
+                                const struct timespec *,
                                 int private) attribute_hidden;
 
+#define lll_timedwait(futex, val, clockid, abstime, private)           \
+  __lll_clocklock_wait (futex, val, clockid, abstime, private)
+
 static inline int
 __attribute__ ((always_inline))
 __lll_clocklock (int *futex, clockid_t clockid,
@@ -87,7 +92,20 @@ __lll_clocklock (int *futex, clockid_t clockid,
   int result = 0;
 
   if (__glibc_unlikely (val != 0))
-    result = __lll_clocklock_wait (futex, clockid, abstime, private);
+    {
+      do
+       {
+         int oldval = atomic_compare_and_exchange_val_24_acq (futex, val, 1);
+         if (oldval != 0)
+           {
+             result = __lll_clocklock_wait (futex, 2, clockid, abstime,
+                                            private);
+             if (result == EINVAL || result == ETIMEDOUT)
+               break;
+           }
+        }
+      while (atomic_compare_and_exchange_val_24_acq (futex, val, 0) != 0);
+    }
   return result;
 }
 #define lll_clocklock(futex, clockid, abstime, private)  \