[RFC PATCH 1/2] Add generic arch support for Requeue PI

Dinakar Guniguntala dino@in.ibm.com
Tue May 5 21:31:00 GMT 2009


Hello,

Please refer to the following mail from Darren Hart on lkml that
adds support for requeue PI implementation in the kernel
http://marc.info/?l=linux-kernel&m=123879118200773&w=2
and some documentation regarding the same
http://marc.info/?l=linux-kernel&m=123922906632577&w=2

The following 2 patches adds glibc support for requeue'ing
from condvars to PI mutexes. This currently has support
only for x86_64 architecture and has been tested pretty
heavily with java and a make subdirs=nptl check

The kernel feature is currently only available in the RT tree,
and I have used the 2.6.29.2-rt11 kernel to test the feature

http://www.kernel.org/pub/linux/kernel/v2.6/linux-2.6.29.2.tar.bz2
http://www.kernel.org/pub/linux/kernel/projects/rt/patch-2.6.29.2-rt11.bz2

I would really appreciate any feedback on this, Thanks!

	-Dinakar


2009-05-05  Dinakar Guniguntala <dino@in.ibm.com>

        * pthread_cond_broadcast.c: Add support for using
        FUTEX_CMP_REQUEUE_PI for PI mutexes.
        * pthread_cond_signal.c: Likewise.
        * pthread_cond_timedwait.c: Add support for using
        FUTEX_WAIT_REQUEUE_PI for PI mutexes.
        * pthread_cond_wait.c: Likewise.


diff -Nurp libc-20090427/nptl/pthread_cond_broadcast.c libc-20090427-1/nptl/pthread_cond_broadcast.c
--- libc-20090427/nptl/pthread_cond_broadcast.c	2007-08-11 14:44:39.000000000 -0400
+++ libc-20090427-1/nptl/pthread_cond_broadcast.c	2009-05-05 15:15:38.000000000 -0400
@@ -1,4 +1,4 @@
-/* Copyright (C) 2003, 2004, 2006, 2007 Free Software Foundation, Inc.
+/* Copyright (C) 2003, 2004, 2006, 2007, 2009 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003.
 
@@ -58,19 +58,28 @@ __pthread_cond_broadcast (cond)
       /* Wake everybody.  */
       pthread_mutex_t *mut = (pthread_mutex_t *) cond->__data.__mutex;
 
-      /* XXX: Kernel so far doesn't support requeue to PI futex.  */
-      /* XXX: Kernel so far can only requeue to the same type of futex,
-	 in this case private (we don't requeue for pshared condvars).  */
+      /* We don't requeue for pshared condvars  */
       if (__builtin_expect (mut->__data.__kind
-			    & (PTHREAD_MUTEX_PRIO_INHERIT_NP
-			       | PTHREAD_MUTEX_PSHARED_BIT), 0))
+			    & PTHREAD_MUTEX_PSHARED_BIT, 0))
 	goto wake_all;
 
-      /* lll_futex_requeue returns 0 for success and non-zero
-	 for errors.  */
-      if (__builtin_expect (lll_futex_requeue (&cond->__data.__futex, 1,
-					       INT_MAX, &mut->__data.__lock,
-					       futex_val, LLL_PRIVATE), 0))
+      if (__builtin_expect (mut->__data.__kind
+			    & PTHREAD_MUTEX_PRIO_INHERIT_NP, 0))
+        {
+         /* lll_futex_requeue_pi returns 0 for success and non-zero
+	    for errors.  */
+         if (__builtin_expect (lll_futex_requeue_pi (&cond->__data.__futex,
+						     1, INT_MAX, 
+						     &mut->__data.__lock,
+					             futex_val, LLL_PRIVATE),
+						     0))
+	   /* The requeue_pi functionality is not available.  */
+	   goto wake_all;
+        }
+      /* lll_futex_requeue returns 0 for success and non-zero for errors.  */
+      else if (__builtin_expect (lll_futex_requeue (&cond->__data.__futex, 1,
+						    INT_MAX, &mut->__data.__lock,
+						    futex_val, LLL_PRIVATE), 0))
 	{
 	  /* The requeue functionality is not available.  */
 	wake_all:
diff -Nurp libc-20090427/nptl/pthread_cond_signal.c libc-20090427-1/nptl/pthread_cond_signal.c
--- libc-20090427/nptl/pthread_cond_signal.c	2007-08-11 14:46:16.000000000 -0400
+++ libc-20090427-1/nptl/pthread_cond_signal.c	2009-05-05 15:16:09.000000000 -0400
@@ -1,4 +1,4 @@
-/* Copyright (C) 2003, 2004, 2007 Free Software Foundation, Inc.
+/* Copyright (C) 2003, 2004, 2007, 2009 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003.
 
@@ -32,8 +32,8 @@ int
 __pthread_cond_signal (cond)
      pthread_cond_t *cond;
 {
-  int pshared = (cond->__data.__mutex == (void *) ~0l)
-		? LLL_SHARED : LLL_PRIVATE;
+  pthread_mutex_t *mutex = (pthread_mutex_t *) cond->__data.__mutex;
+  int pshared = (mutex == (void *) ~0l) ? LLL_SHARED : LLL_PRIVATE;
 
   /* Make sure we are alone.  */
   lll_lock (cond->__data.__lock, pshared);
@@ -44,16 +44,29 @@ __pthread_cond_signal (cond)
       /* Yes.  Mark one of them as woken.  */
       ++cond->__data.__wakeup_seq;
       ++cond->__data.__futex;
+      int futex_val = cond->__data.__futex;
 
+      /* XXX Do we need to ensure that we are not a pshared mutex?  */
+      if (mutex->__data.__kind & PTHREAD_MUTEX_PRIO_INHERIT_NP)
+        {
+	  /* lll_futex_requeue_pi returns 0 for success and non-zero
+	     for errors.  */
+          if (! __builtin_expect (lll_futex_requeue_pi (&cond->__data.__futex,
+				  1, 0, &mutex->__data.__lock, futex_val,
+				  pshared), 0)) 
+            goto done;
+        }
       /* Wake one.  */
-      if (! __builtin_expect (lll_futex_wake_unlock (&cond->__data.__futex, 1,
-						     1, &cond->__data.__lock,
-						     pshared), 0))
+      else if (! __builtin_expect (lll_futex_wake_unlock (&cond->__data.__futex,
+							  1, 1,
+							  &cond->__data.__lock,
+						          pshared), 0))
 	return 0;
 
       lll_futex_wake (&cond->__data.__futex, 1, pshared);
     }
 
+done:
   /* We are done.  */
   lll_unlock (cond->__data.__lock, pshared);
 
diff -Nurp libc-20090427/nptl/pthread_cond_timedwait.c libc-20090427-1/nptl/pthread_cond_timedwait.c
--- libc-20090427/nptl/pthread_cond_timedwait.c	2007-08-13 14:32:59.000000000 -0400
+++ libc-20090427-1/nptl/pthread_cond_timedwait.c	2009-05-05 15:16:21.000000000 -0400
@@ -1,4 +1,4 @@
-/* Copyright (C) 2003, 2004, 2007 Free Software Foundation, Inc.
+/* Copyright (C) 2003, 2004, 2007, 2009 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003.
 
@@ -49,6 +49,7 @@ __pthread_cond_timedwait (cond, mutex, a
   struct _pthread_cleanup_buffer buffer;
   struct _condvar_cleanup_buffer cbuffer;
   int result = 0;
+  int pi_requeued;
 
   /* Catch invalid parameters.  */
   if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000)
@@ -97,6 +98,7 @@ __pthread_cond_timedwait (cond, mutex, a
 
   while (1)
     {
+      pi_requeued = 0;
       struct timespec rt;
       {
 #ifdef __NR_clock_gettime
@@ -155,10 +157,26 @@ __pthread_cond_timedwait (cond, mutex, a
       /* Enable asynchronous cancellation.  Required by the standard.  */
       cbuffer.oldtype = __pthread_enable_asynccancel ();
 
+      if (mutex->__data.__kind & PTHREAD_MUTEX_PRIO_INHERIT_NP)
+	{
+	  /* Try requeueing to the PI mutex, if no support in the kernel
+	     try the non-requeue syscall.  */
+          if (! __builtin_expect ((err = lll_futex_timed_wait_requeue_pi
+				           (&cond->__data.__futex,
+				            futex_val, &rt,
+				            &mutex->__data.__lock,
+				            pshared)), 0))
+	    {
+	       pi_requeued = 1;
+	       goto woken;
+	    }
+	}
+
       /* Wait until woken by signal or broadcast.  */
       err = lll_futex_timed_wait (&cond->__data.__futex,
 				  futex_val, &rt, pshared);
 
+woken:
       /* Disable asynchronous cancellation.  */
       __pthread_disable_asynccancel (cbuffer.oldtype);
 
@@ -208,8 +226,10 @@ __pthread_cond_timedwait (cond, mutex, a
   /* The cancellation handling is back to normal, remove the handler.  */
   __pthread_cleanup_pop (&buffer, 0);
 
-  /* Get the mutex before returning.  */
-  err = __pthread_mutex_cond_lock (mutex);
+  /* Get the mutex before returning. If the requeue_pi call above was successful,
+     the lock is already held in the kernel, so just return to the application.  */
+  if (!pi_requeued)
+    err = __pthread_mutex_cond_lock (mutex);
 
   return err ?: result;
 }
diff -Nurp libc-20090427/nptl/pthread_cond_wait.c libc-20090427-1/nptl/pthread_cond_wait.c
--- libc-20090427/nptl/pthread_cond_wait.c	2007-08-11 14:44:27.000000000 -0400
+++ libc-20090427-1/nptl/pthread_cond_wait.c	2009-05-05 15:16:30.000000000 -0400
@@ -1,4 +1,4 @@
-/* Copyright (C) 2003, 2004, 2006, 2007 Free Software Foundation, Inc.
+/* Copyright (C) 2003, 2004, 2006, 2007, 2009 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003.
 
@@ -100,6 +100,7 @@ __pthread_cond_wait (cond, mutex)
   int err;
   int pshared = (cond->__data.__mutex == (void *) ~0l)
   		? LLL_SHARED : LLL_PRIVATE;
+  int pi_requeued;
 
   /* Make sure we are along.  */
   lll_lock (cond->__data.__lock, pshared);
@@ -142,6 +143,7 @@ __pthread_cond_wait (cond, mutex)
   do
     {
       unsigned int futex_val = cond->__data.__futex;
+      pi_requeued = 0;
 
       /* Prepare to wait.  Release the condvar futex.  */
       lll_unlock (cond->__data.__lock, pshared);
@@ -149,9 +151,25 @@ __pthread_cond_wait (cond, mutex)
       /* Enable asynchronous cancellation.  Required by the standard.  */
       cbuffer.oldtype = __pthread_enable_asynccancel ();
 
-      /* Wait until woken by signal or broadcast.  */
+      if (mutex->__data.__kind & PTHREAD_MUTEX_PRIO_INHERIT_NP)
+        {
+	  /* Try requeueing to the PI mutex, if no support in the kernel
+	     try the non-requeue syscall.  */
+	  if (! __builtin_expect ((lll_futex_wait_requeue_pi
+				     (&cond->__data.__futex,
+				      futex_val,
+				      &mutex->__data.__lock,
+				      pshared)), 0))
+	    {
+	       pi_requeued = 1;
+	       goto woken;
+	    }
+	}
+          
+      /* Wait until woken by signal or broadcast. */
       lll_futex_wait (&cond->__data.__futex, futex_val, pshared);
 
+woken:
       /* Disable asynchronous cancellation.  */
       __pthread_disable_asynccancel (cbuffer.oldtype);
 
@@ -187,7 +205,11 @@ __pthread_cond_wait (cond, mutex)
   /* The cancellation handling is back to normal, remove the handler.  */
   __pthread_cleanup_pop (&buffer, 0);
 
-  /* Get the mutex before returning.  */
+  /* Get the mutex before returning. If the requeue_pi call above was successful,
+     the lock is already held in the kernel, so just return 0 to application.  */
+  if (pi_requeued)
+    return 0;
+
   return __pthread_mutex_cond_lock (mutex);
 }
 
diff -Nurp libc-20090427/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h libc-20090427-1/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
--- libc-20090427/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h	2009-01-02 22:44:45.000000000 -0500
+++ libc-20090427-1/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h	2009-05-04 05:41:38.000000000 -0400
@@ -54,6 +54,8 @@
 #define FUTEX_TRYLOCK_PI	8
 #define FUTEX_WAIT_BITSET	9
 #define FUTEX_WAKE_BITSET	10
+#define FUTEX_WAIT_REQUEUE_PI	11
+#define FUTEX_CMP_REQUEUE_PI	12
 #define FUTEX_PRIVATE_FLAG	128
 #define FUTEX_CLOCK_REALTIME	256
 
@@ -221,6 +223,27 @@ LLL_STUB_UNWIND_INFO_END
   })
 
 
+#define lll_futex_wait_requeue_pi(futex, val, mutex, private) \
+  lll_futex_timed_wait_requeue_pi(futex, val, NULL, mutex, private)
+
+
+#define lll_futex_timed_wait_requeue_pi(futex, val, timeout, mutex, private) \
+  ({									              \
+    register const struct timespec *__to __asm ("r10") = timeout;	              \
+    register void *__mutex __asm  ("r8") = mutex;			              \
+    int __status;							              \
+    register __typeof (val) _val __asm ("edx") = (val);			              \
+    __asm __volatile ("syscall"						              \
+		      : "=a" (__status)					              \
+		      : "0" (SYS_futex), "D" (futex),			              \
+			"S" (__lll_private_flag (FUTEX_WAIT_REQUEUE_PI, private)),    \
+			"d" (_val), "r" (__to),				              \
+			"r" (__mutex)					              \
+		      : "memory", "cc", "r11", "cx");			              \
+    __status;								              \
+  })
+
+
 #define lll_futex_wake(futex, nr, private) \
   do {									      \
     int __ignore;							      \
@@ -552,6 +575,21 @@ LLL_STUB_UNWIND_INFO_END
 		       : "cx", "r11", "cc", "memory");			      \
      __res < 0; })
 
+/* Returns non-zero if error happened, zero if success.  */
+#define lll_futex_requeue_pi(ftx, nr_wake, nr_move, mutex, val, private) \
+  ({ int __res;								      \
+     register int __nr_move __asm ("r10") = nr_move;			      \
+     register void *__mutex __asm ("r8") = mutex;			      \
+     register int __val __asm ("r9") = val;				      \
+     __asm __volatile ("syscall"					      \
+		       : "=a" (__res)					      \
+		       : "0" (__NR_futex), "D" ((void *) ftx),		      \
+			 "S" (__lll_private_flag (FUTEX_CMP_REQUEUE_PI,	      \
+						  private)), "d" (nr_wake),   \
+			 "r" (__nr_move), "r" (__mutex), "r" (__val)	      \
+		       : "cx", "r11", "cc", "memory");			      \
+     __res < 0; })
+
 #define lll_islocked(futex) \
   (futex != LLL_LOCK_INITIALIZER)
 



More information about the Libc-alpha mailing list