This is the mail archive of the libc-alpha@sourceware.org mailing list for the glibc project.


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]
Other format: [Raw text]

[PATCH 2/2] Requeue-PI support for x86 arch


The following patch adds requeue PI support for the x86 arch. This closely
resembles the x86_64 requeue PI support in the current git tree. I have
tested this with 2.6.32-rc6 which should have all of the futex fixes from
Darren Hart and Thomas Gleixner. This applies on top of the updated cfi changes.

The patch is against latest glibc git (2009-Nov-18)

        -Dinakar


2009-11-18  Dinakar Guniguntala <dino@in.ibm.com>

        * sysdeps/unix/sysv/linux/i386/i486/lowlevellock.h: Define
        FUTEX_WAIT_REQUEUE_PI and FUTEX_CMP_REQUEUE_PI.
        * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S: If mutex
        is a PI mutex, then use FUTEX_CMP_REQUEUE_PI.
        * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S: Likewise.
        * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S: If mutex
        is a PI mutex, then use FUTEX_WAIT_REQUEUE_PI.
        * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S: Likewise.


diff -X ignore -Nurp glibc-20091118.cfi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S glibc-20091118.requeue-pi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
--- glibc-20091118.cfi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S	2009-11-18 06:03:11.000000000 -0500
+++ glibc-20091118.requeue-pi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S	2009-11-18 06:02:20.000000000 -0500
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2006, 2007 Free Software Foundation, Inc.
+/* Copyright (C) 2002-2004, 2006-2007, 2009 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -91,12 +91,14 @@ __pthread_cond_broadcast:
 8:	cmpl	$-1, %edi
 	je	9f
 
-	/* XXX: The kernel so far doesn't support requeue to PI futex.  */
-	/* XXX: The kernel only supports FUTEX_CMP_REQUEUE to the same
-	   type of futex (private resp. shared).  */
-	testl	$(PI_BIT | PS_BIT), MUTEX_KIND(%edi)
+	/* Do not use requeue for pshared condvars.  */
+	testl	$PS_BIT, MUTEX_KIND(%edi)
 	jne	9f
 
+	/* Requeue to a PI mutex if the PI bit is set.  */
+	testl	$PI_BIT, MUTEX_KIND(%edi)
+	jne	81f
+
 	/* Wake up all threads.  */
 #ifdef __ASSUME_PRIVATE_FUTEX
 	movl	$(FUTEX_CMP_REQUEUE|FUTEX_PRIVATE_FLAG), %ecx
@@ -138,6 +140,23 @@ __pthread_cond_broadcast:
 
 	cfi_restore_state
 
+81:	movl	$(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+	movl	$SYS_futex, %eax
+	movl	$0x7fffffff, %esi
+	movl	$1, %edx
+	/* Get the address of the futex involved.  */
+# if MUTEX_FUTEX != 0
+	addl	$MUTEX_FUTEX, %edi
+# endif
+	int	$0x80
+
+	/* For any kind of error, which mainly is EAGAIN, we try again
+	with WAKE.  The general test also covers running on old
+	kernels.  */
+	cmpl	$0xfffff001, %eax
+	jb	6b
+	jmp	9f
+
 	.align	16
 	/* Unlock.  */
 4:	LOCK
diff -X ignore -Nurp glibc-20091118.cfi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S glibc-20091118.requeue-pi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
--- glibc-20091118.cfi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S	2009-11-18 04:00:58.000000000 -0500
+++ glibc-20091118.requeue-pi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S	2009-11-18 04:44:49.000000000 -0500
@@ -22,6 +22,7 @@
 #include <lowlevellock.h>
 #include <lowlevelcond.h>
 #include <kernel-features.h>
+#include <pthread-pi-defines.h>
 #include <pthread-errnos.h>
 
 
@@ -86,7 +87,13 @@ __pthread_cond_signal:
 #endif
 	cmpl	$-1, dep_mutex-cond_futex(%ebx)
 	sete	%cl
-	subl	$1, %ecx
+	je	8f
+
+	movl	dep_mutex-cond_futex(%ebx), %edx
+	testl	$PI_BIT, MUTEX_KIND(%edx)
+	jne	9f
+
+8:	subl	$1, %ecx
 #ifdef __ASSUME_PRIVATE_FUTEX
 	andl	$FUTEX_PRIVATE_FLAG, %ecx
 #else
@@ -124,8 +131,34 @@ __pthread_cond_signal:
 
 	cfi_restore_state
 
-7:	/* %ecx should be either FUTEX_WAKE_OP or
-	   FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG from the previous syscall.  */
+9:	movl	$(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+	movl	$SYS_futex, %eax
+	movl	$1, %edx
+	xorl	%esi, %esi
+	movl	dep_mutex-cond_futex(%ebx), %edi
+	movl	(%ebx), %ebp
+	/* FIXME: Until Ingo fixes 4G/4G vDSO, 6 arg syscalls are broken for
+	   sysenter.
+	ENTER_KERNEL  */
+	int	$0x80
+	popl	%ebp
+	popl	%esi
+
+	leal	-cond_futex(%ebx), %edi
+
+	/* For any kind of error, we try again with WAKE.
+	   The general test also covers running on old kernels.  */
+	cmpl	$-4095, %eax
+	jb	4f
+
+7:
+#ifdef __ASSUME_PRIVATE_FUTEX
+	andl	$FUTEX_PRIVATE_FLAG, %ecx
+#else
+	andl	%gs:PRIVATE_FUTEX, %ecx
+#endif
+	orl	$FUTEX_WAKE, %ecx
+
 	xorl	$(FUTEX_WAKE ^ FUTEX_WAKE_OP), %ecx
 	movl	$SYS_futex, %eax
 	/* %edx should be 1 already from $FUTEX_WAKE_OP syscall.
diff -X ignore -Nurp glibc-20091118.cfi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S glibc-20091118.requeue-pi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
--- glibc-20091118.cfi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S	2009-11-18 02:11:27.000000000 -0500
+++ glibc-20091118.requeue-pi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S	2009-11-18 05:19:50.000000000 -0500
@@ -22,6 +22,7 @@
 #include <lowlevellock.h>
 #include <lowlevelcond.h>
 #include <pthread-errnos.h>
+#include <pthread-pi-defines.h>
 #include <kernel-features.h>
 
 
@@ -95,7 +96,7 @@ __pthread_cond_timedwait:
 	addl	$1, cond_futex(%ebx)
 	addl	$(1 << nwaiters_shift), cond_nwaiters(%ebx)
 
-#define FRAME_SIZE 24
+#define FRAME_SIZE 32
 	subl	$FRAME_SIZE, %esp
 	cfi_adjust_cfa_offset(FRAME_SIZE)
 
@@ -107,8 +108,10 @@ __pthread_cond_timedwait:
 	movl	%edx, 16(%esp)
 	movl	%eax, 20(%esp)
 
+	/* Reset the pi-requeued flag.  */
+8:	movl	$0, 24(%esp)
 	/* Get the current time.  */
-8:	movl	%ebx, %edx
+	movl	%ebx, %edx
 #ifdef __NR_clock_gettime
 	/* Get the clock number.  */
 	movl	cond_nwaiters(%ebx), %ebx
@@ -158,6 +161,7 @@ __pthread_cond_timedwait:
 	movl	%edx, 8(%esp)
 
 	movl	cond_futex(%ebx), %edi
+	movl	%edi, 28(%esp)
 
 	/* Unlock.  */
 	LOCK
@@ -172,13 +176,47 @@ __pthread_cond_timedwait:
 4:	call	__pthread_enable_asynccancel
 	movl	%eax, (%esp)
 
-	leal	4(%esp), %esi
 #if FUTEX_PRIVATE_FLAG > 255
 	xorl	%ecx, %ecx
 #endif
 	cmpl	$-1, dep_mutex(%ebx)
 	sete	%cl
-	subl	$1, %ecx
+	je	18f
+
+	movl	dep_mutex(%ebx), %edi
+	/* Requeue to a PI mutex if the PI bit is set.  */
+	testl	$PI_BIT, MUTEX_KIND(%edi)
+	je	18f
+	
+	movl	$(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+	/* The following only works like this because we only support
+	   two clocks, represented using a single bit.  */
+	testl	$1, cond_nwaiters(%ebx)
+	/* XXX Need to implement using sete instead of a jump.  */
+	jne	44f
+	orl	$FUTEX_CLOCK_REALTIME, %ecx
+	
+	/* Requeue-PI uses absolute timeout */
+44:	leal	(%ebp), %esi
+	movl	28(%esp), %edx
+	addl	$cond_futex, %ebx
+	movl	$SYS_futex, %eax
+	ENTER_KERNEL
+	subl	$cond_futex, %ebx
+	movl	%eax, %esi
+	/* Set the pi-requeued flag only if the kernel has returned 0. The
+	   kernel does not hold the mutex on ETIMEDOUT or any other error.  */
+	cmpl	$0, %eax
+	sete	24(%esp)
+	je	19f
+
+	/* Normal and PI futexes dont mix. Use normal futex functions only
+	   if the kernel does not support the PI futex functions.  */
+	cmpl	$-ENOSYS, %eax
+	jne	19f
+	xorl	%ecx, %ecx
+	
+18:	subl	$1, %ecx
 #ifdef __ASSUME_PRIVATE_FUTEX
 	andl	$FUTEX_PRIVATE_FLAG, %ecx
 #else
@@ -187,7 +225,8 @@ __pthread_cond_timedwait:
 #if FUTEX_WAIT != 0
 	addl	$FUTEX_WAIT, %ecx
 #endif
-	movl	%edi, %edx
+	leal	4(%esp), %esi
+	movl	28(%esp), %edx
 	addl	$cond_futex, %ebx
 .Ladd_cond_futex:
 	movl	$SYS_futex, %eax
@@ -196,7 +235,7 @@ __pthread_cond_timedwait:
 .Lsub_cond_futex:
 	movl	%eax, %esi
 
-	movl	(%esp), %eax
+19:	movl	(%esp), %eax
 	call	__pthread_disable_asynccancel
 .LcleanupEND:
 
@@ -284,10 +323,16 @@ __pthread_cond_timedwait:
 #endif
 	jne	10f
 
+11:	xorl	%eax, %eax
+	/* With requeue_pi, the mutex lock is held in the kernel.  */
+	movl	24(%esp), %ecx
+	testl	%ecx, %ecx
+	jnz	26f
+
 	/* Remove cancellation handler.  */
-11:	movl	24+FRAME_SIZE(%esp), %eax
+	movl	24+FRAME_SIZE(%esp), %eax
 	call	__pthread_mutex_cond_lock
-	addl	$FRAME_SIZE, %esp
+26:	addl	$FRAME_SIZE, %esp
 	cfi_adjust_cfa_offset(-FRAME_SIZE);
 
 	/* We return the result of the mutex_lock operation if it failed.  */
diff -X ignore -Nurp glibc-20091118.cfi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S glibc-20091118.requeue-pi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
--- glibc-20091118.cfi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S	2009-11-18 02:11:27.000000000 -0500
+++ glibc-20091118.requeue-pi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S	2009-11-18 05:21:18.000000000 -0500
@@ -22,6 +22,8 @@
 #include <lowlevellock.h>
 #include <lowlevelcond.h>
 #include <tcb-offsets.h>
+#include <pthread-errnos.h>
+#include <pthread-pi-defines.h>
 #include <kernel-features.h>
 
 
@@ -43,6 +45,9 @@ __pthread_cond_wait:
 	cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
 #endif
 
+	pushl	%ebp
+	cfi_adjust_cfa_offset(4)
+	cfi_rel_offset(%ebp, 0)
 	pushl	%edi
 	cfi_adjust_cfa_offset(4)
 	cfi_rel_offset(%edi, 0)
@@ -55,7 +60,7 @@ __pthread_cond_wait:
 	cfi_remember_state
 
 	xorl	%esi, %esi
-	movl	16(%esp), %ebx
+	movl	20(%esp), %ebx
 
 	/* Get internal lock.  */
 	movl	$1, %edx
@@ -71,7 +76,7 @@ __pthread_cond_wait:
 	/* Store the reference to the mutex.  If there is already a
 	   different value in there this is a bad user bug.  */
 2:	cmpl	$-1, dep_mutex(%ebx)
-	movl	20(%esp), %eax
+	movl	24(%esp), %eax
 	je	15f
 	movl	%eax, dep_mutex(%ebx)
 
@@ -87,7 +92,7 @@ __pthread_cond_wait:
 	addl	$1, cond_futex(%ebx)
 	addl	$(1 << nwaiters_shift), cond_nwaiters(%ebx)
 
-#define FRAME_SIZE 16
+#define FRAME_SIZE 20
 	subl	$FRAME_SIZE, %esp
 	cfi_adjust_cfa_offset(FRAME_SIZE)
 
@@ -99,8 +104,10 @@ __pthread_cond_wait:
 	movl	%edx, 8(%esp)
 	movl	%eax, 12(%esp)
 
-8:	movl	cond_futex(%ebx), %edi
-
+	/* Reset the pi-requeued flag.  */
+8:	movl	$0, 16(%esp)
+ 	movl	cond_futex(%ebx), %ebp
+	
 	/* Unlock.  */
 	LOCK
 #if cond_lock == 0
@@ -114,12 +121,36 @@ __pthread_cond_wait:
 4:	call	__pthread_enable_asynccancel
 	movl	%eax, (%esp)
 
-#if FUTEX_PRIVATE_FLAG > 255
 	xorl	%ecx, %ecx
-#endif
 	cmpl	$-1, dep_mutex(%ebx)
 	sete	%cl
-	subl	$1, %ecx
+	je 	18f
+
+	movl	dep_mutex(%ebx), %edi
+	/* Requeue to a PI mutex if the PI bit is set.  */
+	testl	$PI_BIT, MUTEX_KIND(%edi)
+	je	18f
+	
+	movl	$(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+	movl	%ebp, %edx
+	xorl	%esi, %esi
+	addl	$cond_futex, %ebx
+	movl	$SYS_futex, %eax
+	ENTER_KERNEL
+	subl	$cond_futex, %ebx
+	/* Set the pi-requeued flag only if the kernel has returned 0. The
+	   kernel does not hold the mutex on error.  */
+	cmpl	$0, %eax
+	sete	16(%esp)
+	je	19f
+
+	/* Normal and PI futexes dont mix. Use normal futex functions only
+	   if the kernel does not support the PI futex functions.  */
+	cmpl	$-ENOSYS, %eax
+	jne	19f
+	xorl	%ecx, %ecx
+	
+18:	subl	$1, %ecx
 #ifdef __ASSUME_PRIVATE_FUTEX
 	andl	$FUTEX_PRIVATE_FLAG, %ecx
 #else
@@ -128,7 +159,7 @@ __pthread_cond_wait:
 #if FUTEX_WAIT != 0
 	addl	$FUTEX_WAIT, %ecx
 #endif
-	movl	%edi, %edx
+	movl	%ebp, %edx
 	addl	$cond_futex, %ebx
 .Ladd_cond_futex:
 	movl	$SYS_futex, %eax
@@ -136,7 +167,7 @@ __pthread_cond_wait:
 	subl	$cond_futex, %ebx
 .Lsub_cond_futex:
 
-	movl	(%esp), %eax
+19:	movl	(%esp), %eax
 	call	__pthread_disable_asynccancel
 .LcleanupEND:
 
@@ -212,9 +243,15 @@ __pthread_cond_wait:
 #endif
 	jne	10f
 
-11:	movl	20+FRAME_SIZE(%esp), %eax
+	/* With requeue_pi, the mutex lock is held in the kernel.  */
+11:	xorl	%eax, %eax
+	movl	16(%esp), %ecx
+	testl	%ecx, %ecx
+	jnz	20f
+
+	movl	24+FRAME_SIZE(%esp), %eax
 	call	__pthread_mutex_cond_lock
-	addl	$FRAME_SIZE, %esp
+20:	addl	$FRAME_SIZE, %esp
 	cfi_adjust_cfa_offset(-FRAME_SIZE);
 
 14:	popl	%ebx
@@ -226,6 +263,9 @@ __pthread_cond_wait:
 	popl	%edi
 	cfi_adjust_cfa_offset(-4)
 	cfi_restore(%edi)
+	popl	%ebp
+	cfi_adjust_cfa_offset(-4)
+	cfi_restore(%ebp)
 
 	/* We return the result of the mutex_lock operation.  */
 	ret
diff -X ignore -Nurp glibc-20091118.cfi/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h glibc-20091118.requeue-pi/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
--- glibc-20091118.cfi/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h	2009-10-12 00:51:20.000000000 -0400
+++ glibc-20091118.requeue-pi/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h	2009-11-18 04:44:49.000000000 -0500
@@ -54,6 +54,8 @@
 #define FUTEX_TRYLOCK_PI	8
 #define FUTEX_WAIT_BITSET	9
 #define FUTEX_WAKE_BITSET	10
+#define FUTEX_WAIT_REQUEUE_PI	11
+#define FUTEX_CMP_REQUEUE_PI	12
 #define FUTEX_PRIVATE_FLAG	128
 #define FUTEX_CLOCK_REALTIME	256
 


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]