+2009-12-01 Dinakar Guniguntala <dino@in.ibm.com>
+
+ * sysdeps/unix/sysv/linux/i386/i486/lowlevellock.h: Define
+ FUTEX_WAIT_REQUEUE_PI and FUTEX_CMP_REQUEUE_PI.
+ * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S: If mutex
+ is a non robust PI mutex, then use FUTEX_CMP_REQUEUE_PI.
+ * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S: Likewise.
+ * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S: If mutex
+ is a non robust PI mutex, then use FUTEX_WAIT_REQUEUE_PI.
+ * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S: Likewise.
+
2009-12-12 Ulrich Drepper <drepper@redhat.com>
* sysdeps/unix/sysv/linux/i386/i486/sem_timedwait.S (sem_timedwait):
8: cmpl $-1, %edi
je 9f
- /* XXX: The kernel so far doesn't support requeue to PI futex. */
- /* XXX: The kernel only supports FUTEX_CMP_REQUEUE to the same
- type of futex (private resp. shared). */
- testl $(PI_BIT | PS_BIT), MUTEX_KIND(%edi)
+ /* Do not use requeue for pshared condvars. */
+ testl $PS_BIT, MUTEX_KIND(%edi)
jne 9f
+ /* Requeue to a non-robust PI mutex if the PI bit is set and
+ the robust bit is not set. */
+ movl MUTEX_KIND(%edi), %eax
+ andl $(ROBUST_BIT|PI_BIT), %eax
+ cmpl $PI_BIT, %eax
+ je 81f
+
/* Wake up all threads. */
#ifdef __ASSUME_PRIVATE_FUTEX
movl $(FUTEX_CMP_REQUEUE|FUTEX_PRIVATE_FLAG), %ecx
cfi_restore_state
+81: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+ movl $SYS_futex, %eax
+ movl $0x7fffffff, %esi
+ movl $1, %edx
+ /* Get the address of the futex involved. */
+# if MUTEX_FUTEX != 0
+ addl $MUTEX_FUTEX, %edi
+# endif
+ int $0x80
+
+ /* For any kind of error, which mainly is EAGAIN, we try again
+ with WAKE. The general test also covers running on old
+ kernels. */
+ cmpl $0xfffff001, %eax
+ jb 6b
+ jmp 9f
+
/* Initial locking failed. */
1:
#if cond_lock == 0
#include <lowlevellock.h>
#include <lowlevelcond.h>
#include <kernel-features.h>
+#include <pthread-pi-defines.h>
#include <pthread-errnos.h>
#endif
cmpl $-1, dep_mutex-cond_futex(%ebx)
sete %cl
- subl $1, %ecx
+ je 8f
+
+ movl dep_mutex-cond_futex(%ebx), %edx
+ /* Requeue to a non-robust PI mutex if the PI bit is set and
+ the robust bit is not set. */
+ movl MUTEX_KIND(%edx), %eax
+ andl $(ROBUST_BIT|PI_BIT), %eax
+ cmpl $PI_BIT, %eax
+ je 9f
+
+8: subl $1, %ecx
#ifdef __ASSUME_PRIVATE_FUTEX
andl $FUTEX_PRIVATE_FLAG, %ecx
#else
cfi_restore_state
-7: /* %ecx should be either FUTEX_WAKE_OP or
- FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG from the previous syscall. */
+9: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+ movl $SYS_futex, %eax
+ movl $1, %edx
+ xorl %esi, %esi
+ movl dep_mutex-cond_futex(%ebx), %edi
+ movl (%ebx), %ebp
+ /* FIXME: Until Ingo fixes 4G/4G vDSO, 6 arg syscalls are broken for
+ sysenter.
+ ENTER_KERNEL */
+ int $0x80
+ popl %ebp
+ popl %esi
+
+ leal -cond_futex(%ebx), %edi
+
+ /* For any kind of error, we try again with WAKE.
+ The general test also covers running on old kernels. */
+ cmpl $-4095, %eax
+ jb 4f
+
+7:
+#ifdef __ASSUME_PRIVATE_FUTEX
+ andl $FUTEX_PRIVATE_FLAG, %ecx
+#else
+ andl %gs:PRIVATE_FUTEX, %ecx
+#endif
+ orl $FUTEX_WAKE, %ecx
+
xorl $(FUTEX_WAKE ^ FUTEX_WAKE_OP), %ecx
movl $SYS_futex, %eax
/* %edx should be 1 already from $FUTEX_WAKE_OP syscall.
#include <lowlevellock.h>
#include <lowlevelcond.h>
#include <pthread-errnos.h>
+#include <pthread-pi-defines.h>
#include <kernel-features.h>
addl $1, cond_futex(%ebx)
addl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
-#define FRAME_SIZE 24
+#define FRAME_SIZE 32
subl $FRAME_SIZE, %esp
cfi_adjust_cfa_offset(FRAME_SIZE)
movl %edx, 16(%esp)
movl %eax, 20(%esp)
+ /* Reset the pi-requeued flag. */
+8: movl $0, 24(%esp)
/* Get the current time. */
-8: movl %ebx, %edx
+ movl %ebx, %edx
#ifdef __NR_clock_gettime
/* Get the clock number. */
movl cond_nwaiters(%ebx), %ebx
movl %edx, 8(%esp)
movl cond_futex(%ebx), %edi
+ movl %edi, 28(%esp)
/* Unlock. */
LOCK
4: call __pthread_enable_asynccancel
movl %eax, (%esp)
- leal 4(%esp), %esi
#if FUTEX_PRIVATE_FLAG > 255
xorl %ecx, %ecx
#endif
cmpl $-1, dep_mutex(%ebx)
sete %cl
- subl $1, %ecx
+ je 40f
+
+ movl dep_mutex(%ebx), %edi
+ /* Requeue to a non-robust PI mutex if the PI bit is set and
+ the robust bit is not set. */
+ movl MUTEX_KIND(%edi), %eax
+ andl $(ROBUST_BIT|PI_BIT), %eax
+ cmpl $PI_BIT, %eax
+ jne 40f
+
+ movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+ /* The following only works like this because we only support
+ two clocks, represented using a single bit. */
+ testl $1, cond_nwaiters(%ebx)
+ /* XXX Need to implement using sete instead of a jump. */
+ jne 42f
+ orl $FUTEX_CLOCK_REALTIME, %ecx
+
+ /* Requeue-PI uses absolute timeout */
+42: leal (%ebp), %esi
+ movl 28(%esp), %edx
+ addl $cond_futex, %ebx
+ movl $SYS_futex, %eax
+ ENTER_KERNEL
+ subl $cond_futex, %ebx
+ movl %eax, %esi
+ /* Set the pi-requeued flag only if the kernel has returned 0. The
+ kernel does not hold the mutex on ETIMEDOUT or any other error. */
+ cmpl $0, %eax
+ sete 24(%esp)
+ je 41f
+
+ /* Normal and PI futexes dont mix. Use normal futex functions only
+ if the kernel does not support the PI futex functions. */
+ cmpl $-ENOSYS, %eax
+ jne 41f
+ xorl %ecx, %ecx
+
+40: subl $1, %ecx
#ifdef __ASSUME_PRIVATE_FUTEX
andl $FUTEX_PRIVATE_FLAG, %ecx
#else
#if FUTEX_WAIT != 0
addl $FUTEX_WAIT, %ecx
#endif
- movl %edi, %edx
+ leal 4(%esp), %esi
+ movl 28(%esp), %edx
addl $cond_futex, %ebx
.Ladd_cond_futex:
movl $SYS_futex, %eax
.Lsub_cond_futex:
movl %eax, %esi
- movl (%esp), %eax
+41: movl (%esp), %eax
call __pthread_disable_asynccancel
.LcleanupEND:
#endif
jne 10f
+11: xorl %eax, %eax
+ /* With requeue_pi, the mutex lock is held in the kernel. */
+ movl 24(%esp), %ecx
+ testl %ecx, %ecx
+ jnz 26f
+
/* Remove cancellation handler. */
-11: movl 24+FRAME_SIZE(%esp), %eax
+ movl 24+FRAME_SIZE(%esp), %eax
call __pthread_mutex_cond_lock
- addl $FRAME_SIZE, %esp
+26: addl $FRAME_SIZE, %esp
cfi_adjust_cfa_offset(-FRAME_SIZE);
/* We return the result of the mutex_lock operation if it failed. */
cfi_restore_state
+27: call __pthread_mutex_cond_lock_adjust
+ jmp 26b
+
/* Initial locking failed. */
1:
#if cond_lock == 0
#include <lowlevellock.h>
#include <lowlevelcond.h>
#include <tcb-offsets.h>
+#include <pthread-errnos.h>
+#include <pthread-pi-defines.h>
#include <kernel-features.h>
cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
#endif
+ pushl %ebp
+ cfi_adjust_cfa_offset(4)
+ cfi_rel_offset(%ebp, 0)
pushl %edi
cfi_adjust_cfa_offset(4)
cfi_rel_offset(%edi, 0)
cfi_remember_state
xorl %esi, %esi
- movl 16(%esp), %ebx
+ movl 20(%esp), %ebx
/* Get internal lock. */
movl $1, %edx
/* Store the reference to the mutex. If there is already a
different value in there this is a bad user bug. */
2: cmpl $-1, dep_mutex(%ebx)
- movl 20(%esp), %eax
+ movl 24(%esp), %eax
je 15f
movl %eax, dep_mutex(%ebx)
addl $1, cond_futex(%ebx)
addl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
-#define FRAME_SIZE 16
+#define FRAME_SIZE 20
subl $FRAME_SIZE, %esp
cfi_adjust_cfa_offset(FRAME_SIZE)
movl %edx, 8(%esp)
movl %eax, 12(%esp)
-8: movl cond_futex(%ebx), %edi
-
+ /* Reset the pi-requeued flag. */
+8: movl $0, 16(%esp)
+ movl cond_futex(%ebx), %ebp
+
/* Unlock. */
LOCK
#if cond_lock == 0
4: call __pthread_enable_asynccancel
movl %eax, (%esp)
-#if FUTEX_PRIVATE_FLAG > 255
xorl %ecx, %ecx
-#endif
cmpl $-1, dep_mutex(%ebx)
sete %cl
- subl $1, %ecx
+ je 18f
+
+ movl dep_mutex(%ebx), %edi
+ /* Requeue to a non-robust PI mutex if the PI bit is set and
+ the robust bit is not set. */
+ movl MUTEX_KIND(%edi), %eax
+ andl $(ROBUST_BIT|PI_BIT), %eax
+ cmpl $PI_BIT, %eax
+ jne 18f
+
+ movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+ movl %ebp, %edx
+ xorl %esi, %esi
+ addl $cond_futex, %ebx
+ movl $SYS_futex, %eax
+ ENTER_KERNEL
+ subl $cond_futex, %ebx
+ /* Set the pi-requeued flag only if the kernel has returned 0. The
+ kernel does not hold the mutex on error. */
+ cmpl $0, %eax
+ sete 16(%esp)
+ je 19f
+
+ /* Normal and PI futexes dont mix. Use normal futex functions only
+ if the kernel does not support the PI futex functions. */
+ cmpl $-ENOSYS, %eax
+ jne 19f
+ xorl %ecx, %ecx
+
+18: subl $1, %ecx
#ifdef __ASSUME_PRIVATE_FUTEX
andl $FUTEX_PRIVATE_FLAG, %ecx
#else
#if FUTEX_WAIT != 0
addl $FUTEX_WAIT, %ecx
#endif
- movl %edi, %edx
+ movl %ebp, %edx
addl $cond_futex, %ebx
.Ladd_cond_futex:
movl $SYS_futex, %eax
subl $cond_futex, %ebx
.Lsub_cond_futex:
- movl (%esp), %eax
+19: movl (%esp), %eax
call __pthread_disable_asynccancel
.LcleanupEND:
#endif
jne 10f
-11: movl 20+FRAME_SIZE(%esp), %eax
+ /* With requeue_pi, the mutex lock is held in the kernel. */
+11: xorl %eax, %eax
+ movl 16(%esp), %ecx
+ testl %ecx, %ecx
+ jnz 20f
+
+ movl 24+FRAME_SIZE(%esp), %eax
call __pthread_mutex_cond_lock
- addl $FRAME_SIZE, %esp
+20: addl $FRAME_SIZE, %esp
cfi_adjust_cfa_offset(-FRAME_SIZE);
14: popl %ebx
popl %edi
cfi_adjust_cfa_offset(-4)
cfi_restore(%edi)
+ popl %ebp
+ cfi_adjust_cfa_offset(-4)
+ cfi_restore(%ebp)
/* We return the result of the mutex_lock operation. */
ret
cfi_restore_state
+21: call __pthread_mutex_cond_lock_adjust
+ xorl %eax, %eax
+ jmp 20b
+
/* Initial locking failed. */
1:
#if cond_lock == 0
movl $0x7fffffff, %edx
ENTER_KERNEL
-5: movl 20+FRAME_SIZE(%esp), %eax
+5: movl 24+FRAME_SIZE(%esp), %eax
call __pthread_mutex_cond_lock
movl %esi, (%esp)
#define FUTEX_TRYLOCK_PI 8
#define FUTEX_WAIT_BITSET 9
#define FUTEX_WAKE_BITSET 10
+#define FUTEX_WAIT_REQUEUE_PI 11
+#define FUTEX_CMP_REQUEUE_PI 12
#define FUTEX_PRIVATE_FLAG 128
#define FUTEX_CLOCK_REALTIME 256
git://sourceware.org / glibc.git / commitdiff