Optimize x86-64 version of sem_timedwait.

diff --git a/nptl/ChangeLog b/nptl/ChangeLog
index 48fcc0fe091fc43fe750e89e73ecda45d3beed58..5be464e2f22ad23d477e1f22eadfa71a1a88950f 100644 (file)
--- a/nptl/ChangeLog
+++ b/nptl/ChangeLog
@@ -1,5 +1,8 @@
 2009-08-08  Ulrich Drepper  <drepper@redhat.com>
 
+       * sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S (sem_timedwait):
+       Optimize code path used when FUTEX_CLOCK_REALTIME is supported.
+
        * sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
        (__pthread_cond_wait): Optimize by avoiding use of callee-safe
        register.

diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S
index 95762834d3dc5e7e54b089b9eb79a779fac2524c..0291beb169908825d4e3d5bd74562513cd10d236 100644 (file)
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S
@@ -65,34 +65,9 @@ sem_timedwait:
        retq
 
        /* Check whether the timeout value is valid.  */
-1:     pushq   %r12
-       cfi_adjust_cfa_offset(8)
-       cfi_rel_offset(%r12, 0)
-       pushq   %r13
-       cfi_adjust_cfa_offset(8)
-       cfi_rel_offset(%r13, 0)
-       pushq   %r14
-       cfi_adjust_cfa_offset(8)
-       cfi_rel_offset(%r14, 0)
-#ifdef __ASSUME_FUTEX_CLOCK_REALTIME
-# define STACKFRAME 8
-#else
-# define STACKFRAME 24
-#endif
-       subq    $STACKFRAME, %rsp
-       cfi_adjust_cfa_offset(STACKFRAME)
-
-       movq    %rdi, %r12
-       movq    %rsi, %r13
-
-       /* Check for invalid nanosecond field.  */
-       cmpq    $1000000000, 8(%r13)
-       movl    $EINVAL, %r14d
+1:     cmpq    $1000000000, 8(%rsi)
        jae     6f
 
-       LOCK
-       addq    $1, NWAITERS(%r12)
-
 #ifndef __ASSUME_FUTEX_CLOCK_REALTIME
 #  ifdef PIC
        cmpl    $0, __have_futex_clock_realtime(%rip)
@@ -102,15 +77,22 @@ sem_timedwait:
        je      .Lreltmo
 #endif
 
+       /* This push is only needed to store the sem_t pointer for the
+          exception handler.  */
+       pushq   %rdi
+       cfi_adjust_cfa_offset(8)
+
+       movq    %rsi, %r10
+
+       LOCK
+       addq    $1, NWAITERS(%rdi)
+
 .LcleanupSTART:
 13:    call    __pthread_enable_asynccancel
-       movl    %eax, (%rsp)
+       movl    %eax, %r8d
 
-       movq    %r13, %r10
-#if VALUE == 0
-       movq    %r12, %rdi
-#else
-       leaq    VALUE(%r12), %rdi
+#if VALUE != 0
+       leaq    VALUE(%rdi), %rdi
 #endif
        movl    $0xffffffff, %r9d
        movl    $FUTEX_WAIT_BITSET|FUTEX_CLOCK_REALTIME, %esi
@@ -118,22 +100,26 @@ sem_timedwait:
        movl    $SYS_futex, %eax
        xorl    %edx, %edx
        syscall
-       movq    %rax, %r14
+       movq    %rax, %r9
+#if VALUE != 0
+       leaq    -VALUE(%rdi), %rdi
+#endif
 
-       movl    (%rsp), %edi
+       xchgq   %r8, %rdi
        call    __pthread_disable_asynccancel
 .LcleanupEND:
+       movq    %r8, %rdi
 
-       testq   %r14, %r14
+       testq   %r9, %r9
        je      11f
-       cmpq    $-EWOULDBLOCK, %r14
+       cmpq    $-EWOULDBLOCK, %r9
        jne     3f
 
 11:
 #if VALUE == 0
-       movl    (%r12), %eax
+       movl    (%rdi), %eax
 #else
-       movl    VALUE(%r12), %eax
+       movl    VALUE(%rdi), %eax
 #endif
 14:    testl   %eax, %eax
        je      13b
@@ -141,49 +127,74 @@ sem_timedwait:
        leaq    -1(%rax), %rcx
        LOCK
 #if VALUE == 0
-       cmpxchgl %ecx, (%r12)
+       cmpxchgl %ecx, (%rdi)
 #else
-       cmpxchgl %ecx, VALUE(%r12)
+       cmpxchgl %ecx, VALUE(%rdi)
 #endif
        jne     14b
 
-10:    xorl    %eax, %eax
+       xorl    %eax, %eax
 
 15:    LOCK
-       subq    $1, NWAITERS(%r12)
+       subq    $1, NWAITERS(%rdi)
 
-       addq    $STACKFRAME, %rsp
-       cfi_adjust_cfa_offset(-STACKFRAME)
-       popq    %r14
-       cfi_adjust_cfa_offset(-8)
-       cfi_restore(%r14)
-       popq    %r13
+       leaq    8(%rsp), %rsp
        cfi_adjust_cfa_offset(-8)
-       cfi_restore(%r13)
-       popq    %r12
-       cfi_adjust_cfa_offset(-8)
-       cfi_restore(%r12)
        retq
 
-       cfi_adjust_cfa_offset(STACKFRAME + 3 * 8)
-       cfi_rel_offset(%r12, STACKFRAME + 2 * 8)
-       cfi_rel_offset(%r13, STACKFRAME + 1 * 8)
-       cfi_rel_offset(%r14, STACKFRAME)
-3:     negq    %r14
-6:
+       cfi_adjust_cfa_offset(8)
+3:     negq    %r9
 #if USE___THREAD
        movq    errno@gottpoff(%rip), %rdx
-       movl    %r14d, %fs:(%rdx)
+       movl    %r9d, %fs:(%rdx)
 #else
        callq   __errno_location@plt
-       movl    %r14d, (%rax)
+       movl    %r9d, (%rax)
 #endif
 
        orl     $-1, %eax
        jmp     15b
 
+       cfi_adjust_cfa_offset(-8)
+6:
+#if USE___THREAD
+       movq    errno@gottpoff(%rip), %rdx
+       movl    $EINVAL, %fs:(%rdx)
+#else
+       callq   __errno_location@plt
+       movl    $EINVAL, (%rax)
+#endif
+
+       orl     $-1, %eax
+
+       retq
+
 #ifndef __ASSUME_FUTEX_CLOCK_REALTIME
 .Lreltmo:
+       pushq   %r12
+       cfi_adjust_cfa_offset(8)
+       cfi_rel_offset(%r12, 0)
+       pushq   %r13
+       cfi_adjust_cfa_offset(8)
+       cfi_rel_offset(%r13, 0)
+       pushq   %r14
+       cfi_adjust_cfa_offset(8)
+       cfi_rel_offset(%r14, 0)
+
+#ifdef __ASSUME_FUTEX_CLOCK_REALTIME
+# define STACKFRAME 8
+#else
+# define STACKFRAME 24
+#endif
+       subq    $STACKFRAME, %rsp
+       cfi_adjust_cfa_offset(STACKFRAME)
+
+       movq    %rdi, %r12
+       movq    %rsi, %r13
+
+       LOCK
+       addq    $1, NWAITERS(%r12)
+
 7:     xorl    %esi, %esi
        movq    %rsp, %rdi
        movq    $VSYSCALL_ADDR_vgettimeofday, %rax
@@ -202,7 +213,7 @@ sem_timedwait:
        decq    %rdi
 5:     testq   %rdi, %rdi
        movl    $ETIMEDOUT, %r14d
-       js      6b              /* Time is already up.  */
+       js      36f             /* Time is already up.  */
 
        movq    %rdi, (%rsp)    /* Store relative timeout.  */
        movq    %rsi, 8(%rsp)
@@ -235,7 +246,7 @@ sem_timedwait:
        testq   %r14, %r14
        je      9f
        cmpq    $-EWOULDBLOCK, %r14
-       jne     3b
+       jne     33f
 
 9:
 # if VALUE == 0
@@ -254,15 +265,54 @@ sem_timedwait:
        cmpxchgl %ecx, VALUE(%r12)
 # endif
        jne     8b
-       jmp     10b
+
+       xorl    %eax, %eax
+
+45:    LOCK
+       subq    $1, NWAITERS(%r12)
+
+       addq    $STACKFRAME, %rsp
+       cfi_adjust_cfa_offset(-STACKFRAME)
+       popq    %r14
+       cfi_adjust_cfa_offset(-8)
+       cfi_restore(%r14)
+       popq    %r13
+       cfi_adjust_cfa_offset(-8)
+       cfi_restore(%r13)
+       popq    %r12
+       cfi_adjust_cfa_offset(-8)
+       cfi_restore(%r12)
+       retq
+
+       cfi_adjust_cfa_offset(STACKFRAME + 3 * 8)
+       cfi_rel_offset(%r12, STACKFRAME + 2 * 8)
+       cfi_rel_offset(%r13, STACKFRAME + 1 * 8)
+       cfi_rel_offset(%r14, STACKFRAME)
+33:    negq    %r14
+36:
+#if USE___THREAD
+       movq    errno@gottpoff(%rip), %rdx
+       movl    %r14d, %fs:(%rdx)
+#else
+       callq   __errno_location@plt
+       movl    %r14d, (%rax)
 #endif
+
+       orl     $-1, %eax
+       jmp     45b
+#endif
+       cfi_endproc
        .size   sem_timedwait,.-sem_timedwait
 
 
        .type   sem_timedwait_cleanup,@function
 sem_timedwait_cleanup:
+       cfi_startproc
+       cfi_adjust_cfa_offset(8)
+
+       movq    (%rsp), %rdi
        LOCK
-       subq    $1, NWAITERS(%r12)
+       subq    $1, NWAITERS(%rdi)
        movq    %rax, %rdi
 .LcallUR:
        call    _Unwind_Resume@PLT
@@ -272,6 +322,30 @@ sem_timedwait_cleanup:
        .size   sem_timedwait_cleanup,.-sem_timedwait_cleanup
 
 
+#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
+       .type   sem_timedwait_cleanup2,@function
+sem_timedwait_cleanup2:
+       cfi_startproc
+       cfi_adjust_cfa_offset(STACKFRAME + 3 * 8)
+       cfi_rel_offset(%r12, STACKFRAME + 2 * 8)
+       cfi_rel_offset(%r13, STACKFRAME + 1 * 8)
+       cfi_rel_offset(%r14, STACKFRAME)
+
+       LOCK
+       subq    $1, NWAITERS(%r12)
+       movq    %rax, %rdi
+       movq    STACKFRAME(%rsp), %r14
+       movq    STACKFRAME+8(%rsp), %r13
+       movq    STACKFRAME+16(%rsp), %r12
+.LcallUR2:
+       call    _Unwind_Resume@PLT
+       hlt
+.LENDCODE2:
+       cfi_endproc
+       .size   sem_timedwait_cleanup2,.-sem_timedwait_cleanup2
+#endif
+
+
        .section .gcc_except_table,"a",@progbits
 .LexceptSTART:
        .byte   DW_EH_PE_omit                   # @LPStart format
@@ -286,13 +360,19 @@ sem_timedwait_cleanup:
 #ifndef __ASSUME_FUTEX_CLOCK_REALTIME
        .uleb128 .LcleanupSTART2-.LSTARTCODE
        .uleb128 .LcleanupEND2-.LcleanupSTART2
-       .uleb128 sem_timedwait_cleanup-.LSTARTCODE
+       .uleb128 sem_timedwait_cleanup2-.LSTARTCODE
        .uleb128  0
 #endif
        .uleb128 .LcallUR-.LSTARTCODE
        .uleb128 .LENDCODE-.LcallUR
        .uleb128 0
        .uleb128  0
+#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
+       .uleb128 .LcallUR2-.LSTARTCODE
+       .uleb128 .LENDCODE2-.LcallUR2
+       .uleb128 0
+       .uleb128  0
+#endif
 .Lcstend: