This is the mail archive of the
libc-alpha@sourceware.org
mailing list for the glibc project.
[PATCH 2/2] Requeue-PI support for x86 arch
- From: Dinakar Guniguntala <dino at in dot ibm dot com>
- To: libc-alpha at sources dot redhat dot com
- Cc: Ulrich Drepper <drepper at redhat dot com>, Darren Hart <dvhltc at us dot ibm dot com>, Thomas Gleixner <tglx at linutronix dot de>, Richard Henderson <rth at twiddle dot net>
- Date: Wed, 18 Nov 2009 17:41:13 +0530
- Subject: [PATCH 2/2] Requeue-PI support for x86 arch
- References: <20091118120714.GA21633@in.ibm.com>
- Reply-to: dino at in dot ibm dot com
The following patch adds requeue PI support for the x86 arch. This closely
resembles the x86_64 requeue PI support in the current git tree. I have
tested this with 2.6.32-rc6 which should have all of the futex fixes from
Darren Hart and Thomas Gleixner. This applies on top of the updated cfi changes.
The patch is against latest glibc git (2009-Nov-18)
-Dinakar
2009-11-18 Dinakar Guniguntala <dino@in.ibm.com>
* sysdeps/unix/sysv/linux/i386/i486/lowlevellock.h: Define
FUTEX_WAIT_REQUEUE_PI and FUTEX_CMP_REQUEUE_PI.
* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S: If mutex
is a PI mutex, then use FUTEX_CMP_REQUEUE_PI.
* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S: Likewise.
* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S: If mutex
is a PI mutex, then use FUTEX_WAIT_REQUEUE_PI.
* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S: Likewise.
diff -X ignore -Nurp glibc-20091118.cfi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S glibc-20091118.requeue-pi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
--- glibc-20091118.cfi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S 2009-11-18 06:03:11.000000000 -0500
+++ glibc-20091118.requeue-pi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S 2009-11-18 06:02:20.000000000 -0500
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2006, 2007 Free Software Foundation, Inc.
+/* Copyright (C) 2002-2004, 2006-2007, 2009 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
@@ -91,12 +91,14 @@ __pthread_cond_broadcast:
8: cmpl $-1, %edi
je 9f
- /* XXX: The kernel so far doesn't support requeue to PI futex. */
- /* XXX: The kernel only supports FUTEX_CMP_REQUEUE to the same
- type of futex (private resp. shared). */
- testl $(PI_BIT | PS_BIT), MUTEX_KIND(%edi)
+ /* Do not use requeue for pshared condvars. */
+ testl $PS_BIT, MUTEX_KIND(%edi)
jne 9f
+ /* Requeue to a PI mutex if the PI bit is set. */
+ testl $PI_BIT, MUTEX_KIND(%edi)
+ jne 81f
+
/* Wake up all threads. */
#ifdef __ASSUME_PRIVATE_FUTEX
movl $(FUTEX_CMP_REQUEUE|FUTEX_PRIVATE_FLAG), %ecx
@@ -138,6 +140,23 @@ __pthread_cond_broadcast:
cfi_restore_state
+81: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+ movl $SYS_futex, %eax
+ movl $0x7fffffff, %esi
+ movl $1, %edx
+ /* Get the address of the futex involved. */
+# if MUTEX_FUTEX != 0
+ addl $MUTEX_FUTEX, %edi
+# endif
+ int $0x80
+
+ /* For any kind of error, which mainly is EAGAIN, we try again
+ with WAKE. The general test also covers running on old
+ kernels. */
+ cmpl $0xfffff001, %eax
+ jb 6b
+ jmp 9f
+
.align 16
/* Unlock. */
4: LOCK
diff -X ignore -Nurp glibc-20091118.cfi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S glibc-20091118.requeue-pi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
--- glibc-20091118.cfi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S 2009-11-18 04:00:58.000000000 -0500
+++ glibc-20091118.requeue-pi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S 2009-11-18 04:44:49.000000000 -0500
@@ -22,6 +22,7 @@
#include <lowlevellock.h>
#include <lowlevelcond.h>
#include <kernel-features.h>
+#include <pthread-pi-defines.h>
#include <pthread-errnos.h>
@@ -86,7 +87,13 @@ __pthread_cond_signal:
#endif
cmpl $-1, dep_mutex-cond_futex(%ebx)
sete %cl
- subl $1, %ecx
+ je 8f
+
+ movl dep_mutex-cond_futex(%ebx), %edx
+ testl $PI_BIT, MUTEX_KIND(%edx)
+ jne 9f
+
+8: subl $1, %ecx
#ifdef __ASSUME_PRIVATE_FUTEX
andl $FUTEX_PRIVATE_FLAG, %ecx
#else
@@ -124,8 +131,34 @@ __pthread_cond_signal:
cfi_restore_state
-7: /* %ecx should be either FUTEX_WAKE_OP or
- FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG from the previous syscall. */
+9: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+ movl $SYS_futex, %eax
+ movl $1, %edx
+ xorl %esi, %esi
+ movl dep_mutex-cond_futex(%ebx), %edi
+ movl (%ebx), %ebp
+ /* FIXME: Until Ingo fixes 4G/4G vDSO, 6 arg syscalls are broken for
+ sysenter.
+ ENTER_KERNEL */
+ int $0x80
+ popl %ebp
+ popl %esi
+
+ leal -cond_futex(%ebx), %edi
+
+ /* For any kind of error, we try again with WAKE.
+ The general test also covers running on old kernels. */
+ cmpl $-4095, %eax
+ jb 4f
+
+7:
+#ifdef __ASSUME_PRIVATE_FUTEX
+ andl $FUTEX_PRIVATE_FLAG, %ecx
+#else
+ andl %gs:PRIVATE_FUTEX, %ecx
+#endif
+ orl $FUTEX_WAKE, %ecx
+
xorl $(FUTEX_WAKE ^ FUTEX_WAKE_OP), %ecx
movl $SYS_futex, %eax
/* %edx should be 1 already from $FUTEX_WAKE_OP syscall.
diff -X ignore -Nurp glibc-20091118.cfi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S glibc-20091118.requeue-pi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
--- glibc-20091118.cfi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S 2009-11-18 02:11:27.000000000 -0500
+++ glibc-20091118.requeue-pi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S 2009-11-18 05:19:50.000000000 -0500
@@ -22,6 +22,7 @@
#include <lowlevellock.h>
#include <lowlevelcond.h>
#include <pthread-errnos.h>
+#include <pthread-pi-defines.h>
#include <kernel-features.h>
@@ -95,7 +96,7 @@ __pthread_cond_timedwait:
addl $1, cond_futex(%ebx)
addl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
-#define FRAME_SIZE 24
+#define FRAME_SIZE 32
subl $FRAME_SIZE, %esp
cfi_adjust_cfa_offset(FRAME_SIZE)
@@ -107,8 +108,10 @@ __pthread_cond_timedwait:
movl %edx, 16(%esp)
movl %eax, 20(%esp)
+ /* Reset the pi-requeued flag. */
+8: movl $0, 24(%esp)
/* Get the current time. */
-8: movl %ebx, %edx
+ movl %ebx, %edx
#ifdef __NR_clock_gettime
/* Get the clock number. */
movl cond_nwaiters(%ebx), %ebx
@@ -158,6 +161,7 @@ __pthread_cond_timedwait:
movl %edx, 8(%esp)
movl cond_futex(%ebx), %edi
+ movl %edi, 28(%esp)
/* Unlock. */
LOCK
@@ -172,13 +176,47 @@ __pthread_cond_timedwait:
4: call __pthread_enable_asynccancel
movl %eax, (%esp)
- leal 4(%esp), %esi
#if FUTEX_PRIVATE_FLAG > 255
xorl %ecx, %ecx
#endif
cmpl $-1, dep_mutex(%ebx)
sete %cl
- subl $1, %ecx
+ je 18f
+
+ movl dep_mutex(%ebx), %edi
+ /* Requeue to a PI mutex if the PI bit is set. */
+ testl $PI_BIT, MUTEX_KIND(%edi)
+ je 18f
+
+ movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+ /* The following only works like this because we only support
+ two clocks, represented using a single bit. */
+ testl $1, cond_nwaiters(%ebx)
+ /* XXX Need to implement using sete instead of a jump. */
+ jne 44f
+ orl $FUTEX_CLOCK_REALTIME, %ecx
+
+ /* Requeue-PI uses absolute timeout */
+44: leal (%ebp), %esi
+ movl 28(%esp), %edx
+ addl $cond_futex, %ebx
+ movl $SYS_futex, %eax
+ ENTER_KERNEL
+ subl $cond_futex, %ebx
+ movl %eax, %esi
+ /* Set the pi-requeued flag only if the kernel has returned 0. The
+ kernel does not hold the mutex on ETIMEDOUT or any other error. */
+ cmpl $0, %eax
+ sete 24(%esp)
+ je 19f
+
+ /* Normal and PI futexes dont mix. Use normal futex functions only
+ if the kernel does not support the PI futex functions. */
+ cmpl $-ENOSYS, %eax
+ jne 19f
+ xorl %ecx, %ecx
+
+18: subl $1, %ecx
#ifdef __ASSUME_PRIVATE_FUTEX
andl $FUTEX_PRIVATE_FLAG, %ecx
#else
@@ -187,7 +225,8 @@ __pthread_cond_timedwait:
#if FUTEX_WAIT != 0
addl $FUTEX_WAIT, %ecx
#endif
- movl %edi, %edx
+ leal 4(%esp), %esi
+ movl 28(%esp), %edx
addl $cond_futex, %ebx
.Ladd_cond_futex:
movl $SYS_futex, %eax
@@ -196,7 +235,7 @@ __pthread_cond_timedwait:
.Lsub_cond_futex:
movl %eax, %esi
- movl (%esp), %eax
+19: movl (%esp), %eax
call __pthread_disable_asynccancel
.LcleanupEND:
@@ -284,10 +323,16 @@ __pthread_cond_timedwait:
#endif
jne 10f
+11: xorl %eax, %eax
+ /* With requeue_pi, the mutex lock is held in the kernel. */
+ movl 24(%esp), %ecx
+ testl %ecx, %ecx
+ jnz 26f
+
/* Remove cancellation handler. */
-11: movl 24+FRAME_SIZE(%esp), %eax
+ movl 24+FRAME_SIZE(%esp), %eax
call __pthread_mutex_cond_lock
- addl $FRAME_SIZE, %esp
+26: addl $FRAME_SIZE, %esp
cfi_adjust_cfa_offset(-FRAME_SIZE);
/* We return the result of the mutex_lock operation if it failed. */
diff -X ignore -Nurp glibc-20091118.cfi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S glibc-20091118.requeue-pi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
--- glibc-20091118.cfi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S 2009-11-18 02:11:27.000000000 -0500
+++ glibc-20091118.requeue-pi/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S 2009-11-18 05:21:18.000000000 -0500
@@ -22,6 +22,8 @@
#include <lowlevellock.h>
#include <lowlevelcond.h>
#include <tcb-offsets.h>
+#include <pthread-errnos.h>
+#include <pthread-pi-defines.h>
#include <kernel-features.h>
@@ -43,6 +45,9 @@ __pthread_cond_wait:
cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
#endif
+ pushl %ebp
+ cfi_adjust_cfa_offset(4)
+ cfi_rel_offset(%ebp, 0)
pushl %edi
cfi_adjust_cfa_offset(4)
cfi_rel_offset(%edi, 0)
@@ -55,7 +60,7 @@ __pthread_cond_wait:
cfi_remember_state
xorl %esi, %esi
- movl 16(%esp), %ebx
+ movl 20(%esp), %ebx
/* Get internal lock. */
movl $1, %edx
@@ -71,7 +76,7 @@ __pthread_cond_wait:
/* Store the reference to the mutex. If there is already a
different value in there this is a bad user bug. */
2: cmpl $-1, dep_mutex(%ebx)
- movl 20(%esp), %eax
+ movl 24(%esp), %eax
je 15f
movl %eax, dep_mutex(%ebx)
@@ -87,7 +92,7 @@ __pthread_cond_wait:
addl $1, cond_futex(%ebx)
addl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
-#define FRAME_SIZE 16
+#define FRAME_SIZE 20
subl $FRAME_SIZE, %esp
cfi_adjust_cfa_offset(FRAME_SIZE)
@@ -99,8 +104,10 @@ __pthread_cond_wait:
movl %edx, 8(%esp)
movl %eax, 12(%esp)
-8: movl cond_futex(%ebx), %edi
-
+ /* Reset the pi-requeued flag. */
+8: movl $0, 16(%esp)
+ movl cond_futex(%ebx), %ebp
+
/* Unlock. */
LOCK
#if cond_lock == 0
@@ -114,12 +121,36 @@ __pthread_cond_wait:
4: call __pthread_enable_asynccancel
movl %eax, (%esp)
-#if FUTEX_PRIVATE_FLAG > 255
xorl %ecx, %ecx
-#endif
cmpl $-1, dep_mutex(%ebx)
sete %cl
- subl $1, %ecx
+ je 18f
+
+ movl dep_mutex(%ebx), %edi
+ /* Requeue to a PI mutex if the PI bit is set. */
+ testl $PI_BIT, MUTEX_KIND(%edi)
+ je 18f
+
+ movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+ movl %ebp, %edx
+ xorl %esi, %esi
+ addl $cond_futex, %ebx
+ movl $SYS_futex, %eax
+ ENTER_KERNEL
+ subl $cond_futex, %ebx
+ /* Set the pi-requeued flag only if the kernel has returned 0. The
+ kernel does not hold the mutex on error. */
+ cmpl $0, %eax
+ sete 16(%esp)
+ je 19f
+
+ /* Normal and PI futexes dont mix. Use normal futex functions only
+ if the kernel does not support the PI futex functions. */
+ cmpl $-ENOSYS, %eax
+ jne 19f
+ xorl %ecx, %ecx
+
+18: subl $1, %ecx
#ifdef __ASSUME_PRIVATE_FUTEX
andl $FUTEX_PRIVATE_FLAG, %ecx
#else
@@ -128,7 +159,7 @@ __pthread_cond_wait:
#if FUTEX_WAIT != 0
addl $FUTEX_WAIT, %ecx
#endif
- movl %edi, %edx
+ movl %ebp, %edx
addl $cond_futex, %ebx
.Ladd_cond_futex:
movl $SYS_futex, %eax
@@ -136,7 +167,7 @@ __pthread_cond_wait:
subl $cond_futex, %ebx
.Lsub_cond_futex:
- movl (%esp), %eax
+19: movl (%esp), %eax
call __pthread_disable_asynccancel
.LcleanupEND:
@@ -212,9 +243,15 @@ __pthread_cond_wait:
#endif
jne 10f
-11: movl 20+FRAME_SIZE(%esp), %eax
+ /* With requeue_pi, the mutex lock is held in the kernel. */
+11: xorl %eax, %eax
+ movl 16(%esp), %ecx
+ testl %ecx, %ecx
+ jnz 20f
+
+ movl 24+FRAME_SIZE(%esp), %eax
call __pthread_mutex_cond_lock
- addl $FRAME_SIZE, %esp
+20: addl $FRAME_SIZE, %esp
cfi_adjust_cfa_offset(-FRAME_SIZE);
14: popl %ebx
@@ -226,6 +263,9 @@ __pthread_cond_wait:
popl %edi
cfi_adjust_cfa_offset(-4)
cfi_restore(%edi)
+ popl %ebp
+ cfi_adjust_cfa_offset(-4)
+ cfi_restore(%ebp)
/* We return the result of the mutex_lock operation. */
ret
diff -X ignore -Nurp glibc-20091118.cfi/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h glibc-20091118.requeue-pi/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
--- glibc-20091118.cfi/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h 2009-10-12 00:51:20.000000000 -0400
+++ glibc-20091118.requeue-pi/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h 2009-11-18 04:44:49.000000000 -0500
@@ -54,6 +54,8 @@
#define FUTEX_TRYLOCK_PI 8
#define FUTEX_WAIT_BITSET 9
#define FUTEX_WAKE_BITSET 10
+#define FUTEX_WAIT_REQUEUE_PI 11
+#define FUTEX_CMP_REQUEUE_PI 12
#define FUTEX_PRIVATE_FLAG 128
#define FUTEX_CLOCK_REALTIME 256