This is the mail archive of the
libc-alpha@sourceware.org
mailing list for the glibc project.
[PATCH 5/8] Add lock elision to rwlocks
- From: Andi Kleen <andi at firstfloor dot org>
- To: libc-alpha at sourceware dot org
- Cc: Andi Kleen <ak at linux dot intel dot com>
- Date: Thu, 31 Jan 2013 14:29:27 -0800
- Subject: [PATCH 5/8] Add lock elision to rwlocks
- References: <1359671370-19270-1-git-send-email-andi@firstfloor.org>
From: Andi Kleen <ak@linux.intel.com>
This code is very similar to the main pthread mutex code, but we have
to implement part of it in assembler. IFUNC is used to avoid any
fast path overhead on non RTM systems for the lock functions (but not
unlock)
The patch kit is large because there are so many combinations
of reader and writer locks, but it is fairly repetive.
The basic elision algorithm is very similar to normal mutexes:
a wrapper checks if the lock is free (both reader and writer)
We have to check both reader and writer because separating them
would require putting state on different cache lines.
The algorithm is very basic currently and may be tuned
more in the future.
No adaptation at this point, the elision will not automatically
disable itself. The programmer or user can do it explicitely though.
New lock type flags are exposed to force enable/disable elision
for a specific rwlocks.
We use some padding in the internal rwlock as a elision flag.
This is 8 bytes on 64bits currently, but could be made smaller later
without compatibility problems (or to share some bytes with a
adaptation count). 32bit uses one byte.
2013-01-30 Andi Kleen <ak@linux.intel.com>
Hongjiu Lu <hongjiu.lu@intel.com>
* pthread_rwlock_init.c (__pthread_rwlock_init): Support elision.
* pthread_rwlockattr_setkind_np.c (pthread_rwlockattr_setkind_np): Support elision.
* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S: dito.
* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S: dito
* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S: dito.
* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S: dito.
* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S: dito.
* sysdeps/unix/sysv/linux/internaltypes.h (struct pthread_condattr):
(struct pthread_rwlockattr): Support elision flags in internal rwlock.
* sysdeps/unix/sysv/linux/lowlevelrwlock.sym: Add ELIDED.
* sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h: Add elision flags
* sysdeps/unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c: New file
* sysdeps/unix/sysv/linux/x86/pthread_rwlock_trywrlock.c: dito.
* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S: Handle elision.
* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S: dito.
* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S: dito.
* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S: dito.
* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S: dito.
---
nptl/pthread_rwlock_init.c | 4 +
nptl/pthread_rwlock_tryrdlock.c | 2 +
nptl/pthread_rwlock_trywrlock.c | 2 +
nptl/pthread_rwlockattr_setkind_np.c | 12 +++-
.../sysv/linux/i386/i486/pthread_rwlock_rdlock.S | 68 ++++++++++++++++++++
.../linux/i386/i486/pthread_rwlock_timedrdlock.S | 63 ++++++++++++++++++
.../linux/i386/i486/pthread_rwlock_timedwrlock.S | 50 ++++++++++++++-
.../sysv/linux/i386/i486/pthread_rwlock_unlock.S | 19 ++++++
.../sysv/linux/i386/i486/pthread_rwlock_wrlock.S | 56 ++++++++++++++++
nptl/sysdeps/unix/sysv/linux/internaltypes.h | 4 +-
nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym | 5 ++
.../unix/sysv/linux/x86/bits/pthreadtypes.h | 6 +-
.../unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c | 52 +++++++++++++++
.../unix/sysv/linux/x86/pthread_rwlock_trywrlock.c | 55 ++++++++++++++++
.../unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S | 51 +++++++++++++++
.../sysv/linux/x86_64/pthread_rwlock_timedrdlock.S | 51 +++++++++++++++
.../sysv/linux/x86_64/pthread_rwlock_timedwrlock.S | 35 ++++++++++
.../unix/sysv/linux/x86_64/pthread_rwlock_unlock.S | 17 +++++-
.../unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S | 34 ++++++++++
19 files changed, 580 insertions(+), 6 deletions(-)
create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c
create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_trywrlock.c
diff --git a/nptl/pthread_rwlock_init.c b/nptl/pthread_rwlock_init.c
index 16bfe2d..f3680fc 100644
--- a/nptl/pthread_rwlock_init.c
+++ b/nptl/pthread_rwlock_init.c
@@ -68,6 +68,10 @@ __pthread_rwlock_init (rwlock, attr)
header.private_futex));
#endif
+#ifdef __PTHREAD_RWLOCK_ELIDING
+ rwlock->__data.__eliding = iattr->eliding;
+#endif
+
return 0;
}
strong_alias (__pthread_rwlock_init, pthread_rwlock_init)
diff --git a/nptl/pthread_rwlock_tryrdlock.c b/nptl/pthread_rwlock_tryrdlock.c
index 935ac87..896809b 100644
--- a/nptl/pthread_rwlock_tryrdlock.c
+++ b/nptl/pthread_rwlock_tryrdlock.c
@@ -45,4 +45,6 @@ __pthread_rwlock_tryrdlock (pthread_rwlock_t *rwlock)
return result;
}
+#ifndef __pthread_rwlock_tryrdlock
strong_alias (__pthread_rwlock_tryrdlock, pthread_rwlock_tryrdlock)
+#endif
diff --git a/nptl/pthread_rwlock_trywrlock.c b/nptl/pthread_rwlock_trywrlock.c
index 01754ae..6d8a8c4 100644
--- a/nptl/pthread_rwlock_trywrlock.c
+++ b/nptl/pthread_rwlock_trywrlock.c
@@ -38,4 +38,6 @@ __pthread_rwlock_trywrlock (pthread_rwlock_t *rwlock)
return result;
}
+#ifndef __pthread_rwlock_trywrlock
strong_alias (__pthread_rwlock_trywrlock, pthread_rwlock_trywrlock)
+#endif
diff --git a/nptl/pthread_rwlockattr_setkind_np.c b/nptl/pthread_rwlockattr_setkind_np.c
index 64bd341..833e1f0 100644
--- a/nptl/pthread_rwlockattr_setkind_np.c
+++ b/nptl/pthread_rwlockattr_setkind_np.c
@@ -27,12 +27,22 @@ pthread_rwlockattr_setkind_np (attr, pref)
{
struct pthread_rwlockattr *iattr;
+ iattr = (struct pthread_rwlockattr *) attr;
+
+ iattr->eliding = 0;
+ if (pref & (PTHREAD_RWLOCK_ELISION_NP|PTHREAD_RWLOCK_NO_ELISION_NP))
+ {
+ if (pref & PTHREAD_RWLOCK_ELISION_NP)
+ iattr->eliding = 1;
+ if (pref & PTHREAD_RWLOCK_NO_ELISION_NP)
+ iattr->eliding = -1;
+ pref &= ~(PTHREAD_RWLOCK_ELISION_NP|PTHREAD_RWLOCK_NO_ELISION_NP);
+ }
if (pref != PTHREAD_RWLOCK_PREFER_READER_NP
&& pref != PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP
&& __builtin_expect (pref != PTHREAD_RWLOCK_PREFER_WRITER_NP, 0))
return EINVAL;
- iattr = (struct pthread_rwlockattr *) attr;
iattr->lockkind = pref;
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
index 6c46ba6..62c95cb 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
@@ -21,9 +21,16 @@
#include <lowlevelrwlock.h>
#include <pthread-errnos.h>
#include <kernel-features.h>
+#include <hle.h>
#include <stap-probe.h>
+#ifdef PIC
+#define MO(x) x##@GOTOFF(%edx)
+#else
+#define MO(x) x
+#endif
+
.text
.globl __pthread_rwlock_rdlock
@@ -43,6 +50,65 @@ __pthread_rwlock_rdlock:
LIBC_PROBE (rdlock_entry, 1, %ebx)
+#ifdef PIC
+ SETUP_PIC_REG(dx)
+ addl $_GLOBAL_OFFSET_TABLE_,%edx
+#endif
+
+ cmpl $0,MO(__elision_available)
+ jz not_elided_rdlock
+
+ cmpb $0,ELIDING(%ebx)
+ js not_elided_rdlock
+ jnz 2f
+ /* zero: use default */
+
+ cmpl $0,MO(__rwlock_rtm_enabled)
+ jz not_elided_rdlock
+
+2:
+ mov MO(__rwlock_rtm_read_retries),%ecx
+
+try_trans_rdlock:
+ XBEGIN abort_rdlock
+
+ /* Lock writer/reader free? */
+ cmpl $0,WRITER(%ebx)
+ jnz 1f
+ cmpl $0,NR_READERS(%ebx)
+ jnz 1f
+
+ /* Lock is free. Run with transaction */
+ xor %eax,%eax
+
+ pop %ebx
+ cfi_adjust_cfa_offset(-4)
+ pop %esi
+ cfi_adjust_cfa_offset(-4)
+ ret
+
+ /* Lock is not free. Run */
+1: XABORT 0xff
+ jmp not_elided_rdlock
+
+ /* Abort happened. */
+abort_rdlock:
+ testl $_XABORT_CONFLICT,%eax
+ jz not_elided_rdlock
+
+ /* For a reader that aborts due a conflict retry speculation
+ a limited number of times. This way when some reader aborts
+ because the reader count is written the other readers will
+ still elide, at the cost of retrying the speculation. */
+
+ dec %ecx
+ jnz try_trans_rdlock
+
+ /* Otherwise we just fall back directly to the lock.
+ Here's the place to add more adaptation. */
+
+not_elided_rdlock:
+
/* Get the lock. */
movl $1, %edx
xorl %eax, %eax
@@ -188,5 +254,7 @@ __pthread_rwlock_rdlock:
cfi_endproc
.size __pthread_rwlock_rdlock,.-__pthread_rwlock_rdlock
+#ifndef __pthread_rwlock_rdlock
strong_alias (__pthread_rwlock_rdlock, pthread_rwlock_rdlock)
hidden_def (__pthread_rwlock_rdlock)
+#endif
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
index 1908f6f..053624d 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
@@ -21,7 +21,13 @@
#include <lowlevelrwlock.h>
#include <pthread-errnos.h>
#include <kernel-features.h>
+#include <hle.h>
+#ifdef PIC
+#define MO(x) x##@GOTOFF(%edx)
+#else
+#define MO(x) x
+#endif
.text
@@ -48,6 +54,62 @@ pthread_rwlock_timedrdlock:
movl 28(%esp), %ebp
movl 32(%esp), %edi
+#ifdef PIC
+ SETUP_PIC_REG(dx)
+ addl $_GLOBAL_OFFSET_TABLE_,%edx
+#endif
+
+ cmpl $0,MO(__elision_available)
+ jz not_elided_trdlock
+
+ cmpb $0,ELIDING(%ebp)
+ js not_elided_trdlock
+ jnz 2f
+ /* zero: use default */
+
+ cmpl $0,MO(__rwlock_rtm_enabled)
+ jz not_elided_trdlock
+
+2:
+ mov MO(__rwlock_rtm_read_retries),%ecx
+
+try_trans_trdlock:
+ XBEGIN abort_trdlock
+
+ /* Lock writer/reader free? */
+ cmpl $0,WRITER(%ebp)
+ jnz 1f
+ cmpl $0,NR_READERS(%ebp)
+ jnz 1f
+
+ /* Lock is free. Run with transaction */
+ xor %eax,%eax
+ jmp 77f
+
+
+ /* Lock is not free. Run */
+1: XABORT 0xff
+ jmp not_elided_trdlock
+
+ /* Abort happened. */
+abort_trdlock:
+ testl $_XABORT_CONFLICT,%eax
+ jz not_elided_trdlock
+
+ /* For a reader that aborts due a conflict retry speculation
+ a limited number of times. This way when some reader aborts
+ because the reader count is written the other readers will
+ still elide, at the cost of retrying the speculation. */
+
+ dec %ecx
+ jnz try_trans_trdlock
+
+ /* Otherwise we just fall back directly to the lock.
+ Here's the place to add more adaptation. */
+
+not_elided_trdlock:
+
+
/* Get the lock. */
movl $1, %edx
xorl %eax, %eax
@@ -158,6 +220,7 @@ pthread_rwlock_timedrdlock:
7: movl %edx, %eax
+77:
addl $8, %esp
cfi_adjust_cfa_offset(-8)
popl %ebp
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
index e0fc809..6820395 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
@@ -21,7 +21,13 @@
#include <lowlevelrwlock.h>
#include <pthread-errnos.h>
#include <kernel-features.h>
+#include <hle.h>
+#ifdef PIC
+#define MO(x) x##@GOTOFF(%edx)
+#else
+#define MO(x) x
+#endif
.text
@@ -48,6 +54,48 @@ pthread_rwlock_timedwrlock:
movl 28(%esp), %ebp
movl 32(%esp), %edi
+#ifdef PIC
+ SETUP_PIC_REG(dx)
+ addl $_GLOBAL_OFFSET_TABLE_,%edx
+#endif
+
+ cmpl $0,MO(__elision_available)
+ jz not_elided_twrlock
+
+ cmpb $0,ELIDING(%ebp)
+ js not_elided_twrlock
+ jnz try_trans_wrlock
+ /* zero: use default */
+
+ cmpl $0,MO(__rwlock_rtm_enabled)
+ jz not_elided_twrlock
+
+try_trans_wrlock:
+ XBEGIN abort_twrlock
+
+ /* Lock writer free? */
+ cmpl $0,WRITER(%ebp)
+ jnz 1f
+ cmpl $0,NR_READERS(%ebp)
+ jnz 1f
+
+ /* Lock is free. Run with transaction */
+ xor %eax,%eax
+
+ jmp 77f
+
+ /* Lock is not free. Run */
+1: XABORT 0xff
+ jmp not_elided_twrlock
+
+ /* Abort happened. */
+abort_twrlock:
+
+ /* Otherwise we just fall back directly to the lock.
+ Here's the place to add more adaptation. */
+
+not_elided_twrlock:
+
/* Get the lock. */
movl $1, %edx
xorl %eax, %eax
@@ -156,7 +204,7 @@ pthread_rwlock_timedwrlock:
7: movl %edx, %eax
- addl $8, %esp
+77: addl $8, %esp
cfi_adjust_cfa_offset(-8)
popl %ebp
cfi_adjust_cfa_offset(-4)
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
index 708e31c..3169743 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
@@ -20,6 +20,7 @@
#include <lowlevellock.h>
#include <lowlevelrwlock.h>
#include <kernel-features.h>
+#include <hle.h>
.text
@@ -38,6 +39,24 @@ __pthread_rwlock_unlock:
movl 12(%esp), %edi
+ /* Is lock free? */
+ cmpl $0,WRITER(%edi)
+ jnz 1f
+ cmpl $0,NR_READERS(%edi)
+ jnz 1f
+
+ /* Looks free. Assume transaction.
+ If you crash here you unlocked a free lock. */
+ XEND
+ xor %eax,%eax
+
+ pop %edi
+ cfi_adjust_cfa_offset(-4)
+ pop %ebx
+ cfi_adjust_cfa_offset(-4)
+ ret
+
+1:
/* Get the lock. */
movl $1, %edx
xorl %eax, %eax
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
index 6ea17f7..427ba34 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
@@ -21,9 +21,16 @@
#include <lowlevelrwlock.h>
#include <pthread-errnos.h>
#include <kernel-features.h>
+#include <hle.h>
#include <stap-probe.h>
+#ifdef PIC
+#define MO(x) x@GOTOFF(%edx)
+#else
+#define MO(x) x
+#endif
+
.text
.globl __pthread_rwlock_wrlock
@@ -43,6 +50,53 @@ __pthread_rwlock_wrlock:
LIBC_PROBE (wrlock_entry, 1, %ebx)
+#ifdef PIC
+ SETUP_PIC_REG(dx)
+ addl $_GLOBAL_OFFSET_TABLE_,%edx
+#endif
+
+ cmpl $0,MO(__elision_available)
+ jz not_elided_wrlock
+
+ cmpb $0,ELIDING(%ebx)
+ js not_elided_wrlock
+ jnz try_trans_wrlock
+ /* zero: use default */
+
+ cmpl $0,MO(__rwlock_rtm_enabled)
+ jz not_elided_wrlock
+
+try_trans_wrlock:
+ XBEGIN abort_wrlock
+
+ /* Lock writer free? */
+ /* Ignore readers because we don't need them */
+ cmpl $0,WRITER(%ebx)
+ jnz 1f
+ cmpl $0,NR_READERS(%ebx)
+ jnz 1f
+
+ /* Lock is free. Run with transaction */
+ xor %eax,%eax
+
+ pop %ebx
+ cfi_adjust_cfa_offset(-4)
+ pop %esi
+ cfi_adjust_cfa_offset(-4)
+ ret
+
+ /* Lock is not free. Run */
+1: XABORT 0xff
+ jmp not_elided_wrlock
+
+ /* Abort happened. */
+abort_wrlock:
+
+ /* Otherwise we just fall back directly to the lock.
+ Here's the place to add more adaptation. */
+
+not_elided_wrlock:
+
/* Get the lock. */
movl $1, %edx
xorl %eax, %eax
@@ -179,5 +233,7 @@ __pthread_rwlock_wrlock:
cfi_endproc
.size __pthread_rwlock_wrlock,.-__pthread_rwlock_wrlock
+#ifndef __pthread_rwlock_wrlock
strong_alias (__pthread_rwlock_wrlock, pthread_rwlock_wrlock)
hidden_def (__pthread_rwlock_wrlock)
+#endif
diff --git a/nptl/sysdeps/unix/sysv/linux/internaltypes.h b/nptl/sysdeps/unix/sysv/linux/internaltypes.h
index 699a618..5d76107 100644
--- a/nptl/sysdeps/unix/sysv/linux/internaltypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/internaltypes.h
@@ -86,7 +86,9 @@ struct pthread_condattr
struct pthread_rwlockattr
{
int lockkind;
- int pshared;
+ short pshared;
+ char eliding;
+ char pad;
};
diff --git a/nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym b/nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym
index f50b25b..fcbcd5a 100644
--- a/nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym
+++ b/nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym
@@ -3,6 +3,10 @@
#include <bits/pthreadtypes.h>
#include <bits/wordsize.h>
+#ifndef __PTHREAD_RWLOCK_ELIDING
+#define __eliding __shared
+#endif
+
--
MUTEX offsetof (pthread_rwlock_t, __data.__lock)
@@ -14,3 +18,4 @@ WRITERS_QUEUED offsetof (pthread_rwlock_t, __data.__nr_writers_queued)
FLAGS offsetof (pthread_rwlock_t, __data.__flags)
WRITER offsetof (pthread_rwlock_t, __data.__writer)
PSHARED offsetof (pthread_rwlock_t, __data.__shared)
+ELIDING offsetof (pthread_rwlock_t, __data.__eliding)
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h
index 1852e07..21b127d 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h
@@ -183,12 +183,13 @@ typedef union
unsigned int __nr_writers_queued;
int __writer;
int __shared;
- unsigned long int __pad1;
+ long __eliding;
unsigned long int __pad2;
/* FLAGS must stay at this position in the structure to maintain
binary compatibility. */
unsigned int __flags;
# define __PTHREAD_RWLOCK_INT_FLAGS_SHARED 1
+# define __PTHREAD_RWLOCK_ELIDING 1
} __data;
# else
struct
@@ -203,8 +204,9 @@ typedef union
binary compatibility. */
unsigned char __flags;
unsigned char __shared;
- unsigned char __pad1;
+ char __eliding;
unsigned char __pad2;
+# define __PTHREAD_RWLOCK_ELIDING 2
int __writer;
} __data;
# endif
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c b/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c
new file mode 100644
index 0000000..1e2a2bb
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c
@@ -0,0 +1,52 @@
+/* pthread_rwlock_tryrdlock: Lock eliding version of pthreads rwlock_tryrdlock.
+ Copyright (C) 2013 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+#include <pthread.h>
+#include <pthreadP.h>
+#include <hle.h>
+#include "elision-conf.h"
+#include "init-arch.h"
+
+#define __pthread_rwlock_tryrdlock __full_pthread_rwlock_tryrdlock
+#include <nptl/pthread_rwlock_tryrdlock.c>
+#undef __pthread_rwlock_tryrdlock
+
+int
+__pthread_rwlock_tryrdlock(pthread_rwlock_t *rwlock)
+{
+ unsigned status;
+
+ if ((rwlock->__data.__eliding > 0 && __elision_available)
+ || (rwlock->__data.__eliding == 0 && __rwlock_rtm_enabled))
+ {
+ if ((status = _xbegin()) == _XBEGIN_STARTED)
+ {
+ if (rwlock->__data.__writer == 0
+ && rwlock->__data.__nr_readers == 0)
+ return 0;
+ /* Lock was busy. Fall back to normal locking.
+ Could also _xend here but xabort with 0xff code
+ is more visible in the profiler. */
+ _xabort (0xff);
+ }
+ /* Aborts come here */
+ }
+
+ return __full_pthread_rwlock_tryrdlock (rwlock);
+}
+
+strong_alias(__pthread_rwlock_tryrdlock, pthread_rwlock_tryrdlock);
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_trywrlock.c b/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_trywrlock.c
new file mode 100644
index 0000000..e5f78ad
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_trywrlock.c
@@ -0,0 +1,55 @@
+/* pthread_rwlock_trywrlock: Lock eliding version of pthreads rwlock trywrlock.
+ Copyright (C) 2013 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+#include <pthread.h>
+#include <pthreadP.h>
+#include <hle.h>
+#include "elision-conf.h"
+#include "init-arch.h"
+
+#define __pthread_rwlock_trywrlock __full_pthread_rwlock_trywrlock
+#include <nptl/pthread_rwlock_trywrlock.c>
+#undef __pthread_rwlock_trywrlock
+
+int
+__pthread_rwlock_trywrlock(pthread_rwlock_t *rwlock)
+{
+ unsigned status;
+ int elision = 0;
+
+ if (rwlock->__data.__eliding == 0 && __rwlock_rtm_enabled)
+ {
+ _xabort (0xfd);
+ elision = 1;
+ }
+ if (elision || (rwlock->__data.__eliding > 0 && __elision_available))
+ {
+ if ((status = _xbegin()) == _XBEGIN_STARTED)
+ {
+ if (rwlock->__data.__writer == 0
+ && rwlock->__data.__nr_readers == 0)
+ return 0;
+ /* Lock was busy. Fall back to normal locking.
+ Could also _xend here but xabort with 0xff code
+ is more visible in the profiler. */
+ _xabort (0xff);
+ }
+ }
+
+ return __full_pthread_rwlock_trywrlock (rwlock);
+}
+strong_alias(__pthread_rwlock_trywrlock, pthread_rwlock_trywrlock);
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S
index 7681818..11f0ba1 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S
@@ -22,6 +22,7 @@
#include <pthread-errnos.h>
#include <kernel-features.h>
#include <stap-probe.h>
+#include <hle.h>
.text
@@ -33,6 +34,54 @@ __pthread_rwlock_rdlock:
LIBC_PROBE (rdlock_entry, 1, %rdi)
+ cmpl $0,__elision_available(%rip)
+ jz not_elided_rdlock
+
+ cmpq $0,ELIDING(%rdi)
+ js not_elided_rdlock
+ jnz 2f
+ /* zero: use default */
+
+ cmpl $0,__rwlock_rtm_enabled(%rip)
+ jz not_elided_rdlock
+
+2:
+ mov __rwlock_rtm_read_retries(%rip),%esi
+
+try_trans_rdlock:
+ XBEGIN abort_rdlock
+
+ /* Lock reader/writer free? */
+ cmpl $0,WRITER(%rdi)
+ jnz 1f
+ cmpl $0,NR_READERS(%rdi)
+ jnz 1f
+
+ /* Lock is free. Run with transaction */
+ xor %eax,%eax
+ ret
+
+ /* Lock is not free. Run */
+1: XABORT 0xff
+ jmp not_elided_rdlock
+
+ /* Abort happened. */
+abort_rdlock:
+ testl $_XABORT_CONFLICT,%eax
+ jz not_elided_rdlock
+
+ /* For a reader that aborts due a conflict retry speculation
+ a limited number of times. This way when some reader aborts
+ because the reader count is written the other readers will
+ still elide, at the cost of retrying the speculation. */
+
+ dec %esi
+ jnz try_trans_rdlock
+
+ /* Otherwise we just fall back directly to the lock.
+ Here's the place to add more adaptation. */
+
+not_elided_rdlock:
xorq %r10, %r10
/* Get the lock. */
@@ -173,5 +222,7 @@ __pthread_rwlock_rdlock:
cfi_endproc
.size __pthread_rwlock_rdlock,.-__pthread_rwlock_rdlock
+#ifndef __pthread_rwlock_rdlock
strong_alias (__pthread_rwlock_rdlock, pthread_rwlock_rdlock)
hidden_def (__pthread_rwlock_rdlock)
+#endif
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S
index 57fe1e9..796fa0e 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S
@@ -21,6 +21,7 @@
#include <lowlevelrwlock.h>
#include <pthread-errnos.h>
#include <kernel-features.h>
+#include <hle.h>
.text
@@ -29,6 +30,56 @@
.align 16
pthread_rwlock_timedrdlock:
cfi_startproc
+
+ cmpl $0,__elision_available(%rip)
+ jz not_elided_timedrdlock
+
+ cmpq $0,ELIDING(%rdi)
+ js not_elided_timedrdlock
+ jnz 2f
+ /* zero: use default */
+
+ cmpl $0,__rwlock_rtm_enabled(%rip)
+ jz not_elided_timedrdlock
+
+2:
+ mov __rwlock_rtm_read_retries(%rip),%ecx
+
+try_trans_timedrdlock:
+ XBEGIN abort_timedrdlock
+
+ /* Lock reader/writer free? */
+ cmpl $0,WRITER(%rdi)
+ jnz 1f
+ cmpl $0,NR_READERS(%rdi)
+ jnz 1f
+
+ /* Lock is free. Run with transaction */
+ xor %eax,%eax
+ ret
+
+ /* Lock is not free. Run */
+1: XABORT 0xff
+ jmp not_elided_timedrdlock
+
+ /* Abort happened. */
+abort_timedrdlock:
+ testl $_XABORT_CONFLICT,%eax
+ jz not_elided_timedrdlock
+
+ /* For a reader that aborts due a conflict retry speculation
+ a limited number of times. This way when some reader aborts
+ because the reader count is written the other readers will
+ still elide, at the cost of retrying the speculation. */
+
+ dec %ecx
+ jnz try_trans_timedrdlock
+
+ /* Otherwise we just fall back directly to the lock.
+ Here's the place to add more adaptation. */
+
+not_elided_timedrdlock:
+
pushq %r12
cfi_adjust_cfa_offset(8)
cfi_rel_offset(%r12, 0)
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S
index 391be17..925e2f8 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S
@@ -21,6 +21,7 @@
#include <lowlevelrwlock.h>
#include <pthread-errnos.h>
#include <kernel-features.h>
+#include <hle.h>
.text
@@ -29,6 +30,40 @@
.align 16
pthread_rwlock_timedwrlock:
cfi_startproc
+
+ cmpl $0,__elision_available(%rip)
+ jz not_elided_timedwrlock
+
+ cmpq $0,ELIDING(%rdi)
+ js not_elided_timedwrlock
+ jnz 2f
+ /* zero: use default */
+
+ cmpl $0,__rwlock_rtm_enabled(%rip)
+ jz not_elided_timedwrlock
+
+2:
+ XBEGIN abort
+
+ /* Lock free? */
+ cmpl $0,NR_READERS(%rdi)
+ jnz 1f
+ cmpl $0,WRITER(%rdi)
+ jnz 1f
+
+ /* Lock is free. Run with transaction */
+ xor %rax, %rax
+ ret
+
+ /* Lock is not free. Run */
+1: XABORT 0xff
+ jmp not_elided_timedwrlock
+
+ /* Abort happened. */
+abort:
+ /* No retries here */
+
+not_elided_timedwrlock:
pushq %r12
cfi_adjust_cfa_offset(8)
cfi_rel_offset(%r12, 0)
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
index 86dda05..a9876f2 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
@@ -20,7 +20,7 @@
#include <lowlevellock.h>
#include <lowlevelrwlock.h>
#include <kernel-features.h>
-
+#include <hle.h>
.text
@@ -29,6 +29,21 @@
.align 16
__pthread_rwlock_unlock:
cfi_startproc
+
+ /* Is lock free? */
+ cmpl $0,WRITER(%rdi)
+ jnz 1f
+ cmpl $0,NR_READERS(%rdi)
+ jnz 1f
+
+ /* Looks free. Assume transaction.
+ If you crash here you unlocked a free lock. */
+ XEND
+ xor %rax,%rax
+ ret
+
+1:
+
/* Get the lock. */
movl $1, %esi
xorl %eax, %eax
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S
index 734bee3..84dac09 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S
@@ -22,6 +22,7 @@
#include <pthread-errnos.h>
#include <kernel-features.h>
#include <stap-probe.h>
+#include <hle.h>
.text
@@ -33,6 +34,37 @@ __pthread_rwlock_wrlock:
LIBC_PROBE (wrlock_entry, 1, %rdi)
+ cmpl $0,__elision_available(%rip)
+ jz not_elided_wrlock
+
+ cmpq $0,ELIDING(%rdi)
+ js not_elided_wrlock
+ jnz 2f
+ /* zero: use default */
+
+ cmpl $0,__rwlock_rtm_enabled(%rip)
+ jz not_elided_wrlock
+
+2:
+ XBEGIN abort_wrlock
+
+ /* Lock free? */
+ cmpl $0,WRITER(%rdi)
+ jnz 1f
+ cmpl $0,NR_READERS(%rdi)
+ jnz 1f
+
+ /* Lock is free. Run with transaction */
+ xor %eax,%eax
+ ret
+
+ /* Lock is not free. End transaction */
+1: XABORT 0xff
+ jmp not_elided_wrlock
+
+ /* Abort happened. */
+abort_wrlock:
+not_elided_wrlock:
xorq %r10, %r10
/* Get the lock. */
@@ -161,5 +193,7 @@ __pthread_rwlock_wrlock:
cfi_endproc
.size __pthread_rwlock_wrlock,.-__pthread_rwlock_wrlock
+#ifndef __pthread_rwlock_wrlock
strong_alias (__pthread_rwlock_wrlock, pthread_rwlock_wrlock)
hidden_def (__pthread_rwlock_wrlock)
+#endif
--
1.7.7.6