This is the mail archive of the
libc-alpha@sourceware.org
mailing list for the glibc project.
Re: [PATCH 05/14] Add lock elision to rwlocks
- From: "Carlos O'Donell" <carlos at redhat dot com>
- To: Andi Kleen <andi at firstfloor dot org>, Torvald Riegel <triegel at redhat dot com>, Rich Felker <dalias at aerifal dot cx>
- Cc: libc-alpha at sourceware dot org, Andi Kleen <ak at linux dot jf dot intel dot com>
- Date: Fri, 28 Jun 2013 03:34:17 -0400
- Subject: Re: [PATCH 05/14] Add lock elision to rwlocks
- References: <1372398717-16530-1-git-send-email-andi at firstfloor dot org> <1372398717-16530-6-git-send-email-andi at firstfloor dot org>
I need some consensus on this issue.
In POSIX Issue 7:
http://pubs.opengroup.org/onlinepubs/9699919799/functions/pthread_rwlock_trywrlock.html
The language for pthread_rlock_wrlock and pthread_rwlock_trywrlock says
that it "may" deadlock, or it "may" also return EDEADLK.
My reading of this is that Torvald's guideline is incorrect:
http://sourceware.org/glibc/wiki/LockElisionGuide#Compability_with_pthreads_rwlock
In that wrlock/trywrlock are not *required* to have a deadlock on relock
semantic by the standard.
The same soft language of "may deadlock" is used for rdlock and thus the same
flexibility is granted there.
Therefore this patch #5 falls within the semantic guarantees of the standard
and therefore can be considered correct.
Comments?
> 2013-06-24 Andi Kleen <ak@linux.intel.com>
> Hongjiu Lu <hongjiu.lu@intel.com>
>
> * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S: dito.
> * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S: dito
> * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S: dito.
> * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S: dito.
> * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S: dito.
> * sysdeps/unix/sysv/linux/internaltypes.h (struct pthread_condattr):
> (struct pthread_rwlockattr): Support elision flags in internal rwlock.
> * sysdeps/unix/sysv/linux/lowlevelrwlock.sym: Add RW_ELISION.
> * sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h: Add elision flags
> * sysdeps/unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c: New file
> * sysdeps/unix/sysv/linux/x86/pthread_rwlock_trywrlock.c: dito.
> * sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S: Handle elision.
> * sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S: dito.
> * sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S: dito.
> * sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S: dito.
> * sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S: dito.
> ---
> nptl/pthread_rwlock_tryrdlock.c | 2 +
> nptl/pthread_rwlock_trywrlock.c | 2 +
> .../sysv/linux/i386/i486/pthread_rwlock_rdlock.S | 65 ++++++++++++++++++++++
> .../linux/i386/i486/pthread_rwlock_timedrdlock.S | 60 ++++++++++++++++++++
> .../linux/i386/i486/pthread_rwlock_timedwrlock.S | 46 ++++++++++++++-
> .../sysv/linux/i386/i486/pthread_rwlock_unlock.S | 19 +++++++
> .../sysv/linux/i386/i486/pthread_rwlock_wrlock.S | 52 +++++++++++++++++
> nptl/sysdeps/unix/sysv/linux/internaltypes.h | 16 +++++-
> nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym | 5 ++
> .../unix/sysv/linux/x86/bits/pthreadtypes.h | 6 +-
> .../unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c | 52 +++++++++++++++++
> .../unix/sysv/linux/x86/pthread_rwlock_trywrlock.c | 51 +++++++++++++++++
> .../unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S | 48 ++++++++++++++++
> .../sysv/linux/x86_64/pthread_rwlock_timedrdlock.S | 48 ++++++++++++++++
> .../sysv/linux/x86_64/pthread_rwlock_timedwrlock.S | 32 +++++++++++
> .../unix/sysv/linux/x86_64/pthread_rwlock_unlock.S | 17 +++++-
> .../unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S | 31 +++++++++++
> 17 files changed, 546 insertions(+), 6 deletions(-)
> create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c
> create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_trywrlock.c
>
> diff --git a/nptl/pthread_rwlock_tryrdlock.c b/nptl/pthread_rwlock_tryrdlock.c
> index 935ac87..896809b 100644
> --- a/nptl/pthread_rwlock_tryrdlock.c
> +++ b/nptl/pthread_rwlock_tryrdlock.c
> @@ -45,4 +45,6 @@ __pthread_rwlock_tryrdlock (pthread_rwlock_t *rwlock)
>
> return result;
> }
> +#ifndef __pthread_rwlock_tryrdlock
> strong_alias (__pthread_rwlock_tryrdlock, pthread_rwlock_tryrdlock)
> +#endif
> diff --git a/nptl/pthread_rwlock_trywrlock.c b/nptl/pthread_rwlock_trywrlock.c
> index 01754ae..6d8a8c4 100644
> --- a/nptl/pthread_rwlock_trywrlock.c
> +++ b/nptl/pthread_rwlock_trywrlock.c
> @@ -38,4 +38,6 @@ __pthread_rwlock_trywrlock (pthread_rwlock_t *rwlock)
>
> return result;
> }
> +#ifndef __pthread_rwlock_trywrlock
> strong_alias (__pthread_rwlock_trywrlock, pthread_rwlock_trywrlock)
> +#endif
> diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
> index 6c46ba6..4151b03 100644
> --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
> +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
> @@ -21,9 +21,16 @@
> #include <lowlevelrwlock.h>
> #include <pthread-errnos.h>
> #include <kernel-features.h>
> +#include <hle.h>
>
> #include <stap-probe.h>
>
> +#ifdef PIC
> +#define MO(x) x##@GOTOFF(%edx)
> +#else
> +#define MO(x) x
> +#endif
> +
> .text
>
> .globl __pthread_rwlock_rdlock
> @@ -43,6 +50,62 @@ __pthread_rwlock_rdlock:
>
> LIBC_PROBE (rdlock_entry, 1, %ebx)
>
> +#ifdef PIC
> + SETUP_PIC_REG(dx)
> + addl $_GLOBAL_OFFSET_TABLE_,%edx
> +#endif
> +
> + cmpb $0,RW_ELISION(%ebx)
> + js not_elided_rdlock
> + jnz 2f
> + /* zero: use default */
> +
> + cmpl $0,MO(__rwlock_rtm_enabled)
> + jz not_elided_rdlock
> +
> +2:
> + mov MO(__rwlock_rtm_read_retries),%ecx
> +
> +try_trans_rdlock:
> + XBEGIN abort_rdlock
> +
> + /* Lock writer/reader free? */
> + cmpl $0,WRITER(%ebx)
> + jnz 1f
> + cmpl $0,NR_READERS(%ebx)
> + jnz 1f
> +
> + /* Lock is free. Run with transaction */
> + xor %eax,%eax
> +
> + pop %ebx
> + cfi_adjust_cfa_offset(-4)
> + pop %esi
> + cfi_adjust_cfa_offset(-4)
> + ret
> +
> + /* Lock is not free. Run */
> +1: XABORT 0xff
> + jmp not_elided_rdlock
> +
> + /* Abort happened. */
> +abort_rdlock:
> + testl $_XABORT_CONFLICT,%eax
> + jz not_elided_rdlock
> +
> + /* For a reader that aborts due a conflict retry speculation
> + a limited number of times. This way when some reader aborts
> + because the reader count is written the other readers will
> + still elide, at the cost of retrying the speculation. */
> +
> + dec %ecx
> + jnz try_trans_rdlock
> +
> + /* Otherwise we just fall back directly to the lock.
> + Here's the place to add more adaptation. */
> +
> +not_elided_rdlock:
> +
> /* Get the lock. */
> movl $1, %edx
> xorl %eax, %eax
> @@ -188,5 +251,7 @@ __pthread_rwlock_rdlock:
> cfi_endproc
> .size __pthread_rwlock_rdlock,.-__pthread_rwlock_rdlock
>
> +#ifndef __pthread_rwlock_rdlock
> strong_alias (__pthread_rwlock_rdlock, pthread_rwlock_rdlock)
> hidden_def (__pthread_rwlock_rdlock)
> +#endif
> diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
> index 1908f6f..e710c49 100644
> --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
> +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
> @@ -21,7 +21,13 @@
> #include <lowlevelrwlock.h>
> #include <pthread-errnos.h>
> #include <kernel-features.h>
> +#include <hle.h>
>
> +#ifdef PIC
> +#define MO(x) x##@GOTOFF(%edx)
> +#else
> +#define MO(x) x
> +#endif
>
> .text
>
> @@ -48,6 +54,59 @@ pthread_rwlock_timedrdlock:
> movl 28(%esp), %ebp
> movl 32(%esp), %edi
>
> +#ifdef PIC
> + SETUP_PIC_REG(dx)
> + addl $_GLOBAL_OFFSET_TABLE_,%edx
> +#endif
> +
> + cmpb $0,RW_ELISION(%ebp)
> + js not_elided_trdlock
> + jnz 2f
> + /* zero: use default */
> +
> + cmpl $0,MO(__rwlock_rtm_enabled)
> + jz not_elided_trdlock
> +
> +2:
> + mov MO(__rwlock_rtm_read_retries),%ecx
> +
> +try_trans_trdlock:
> + XBEGIN abort_trdlock
> +
> + /* Lock writer/reader free? */
> + cmpl $0,WRITER(%ebp)
> + jnz 1f
> + cmpl $0,NR_READERS(%ebp)
> + jnz 1f
> +
> + /* Lock is free. Run with transaction */
> + xor %eax,%eax
> + jmp 77f
> +
> +
> + /* Lock is not free. Run */
> +1: XABORT 0xff
> + jmp not_elided_trdlock
> +
> + /* Abort happened. */
> +abort_trdlock:
> + testl $_XABORT_CONFLICT,%eax
> + jz not_elided_trdlock
> +
> + /* For a reader that aborts due a conflict retry speculation
> + a limited number of times. This way when some reader aborts
> + because the reader count is written the other readers will
> + still elide, at the cost of retrying the speculation. */
> +
> + dec %ecx
> + jnz try_trans_trdlock
> +
> + /* Otherwise we just fall back directly to the lock.
> + Here's the place to add more adaptation. */
> +
> +not_elided_trdlock:
> +
> +
> /* Get the lock. */
> movl $1, %edx
> xorl %eax, %eax
> @@ -158,6 +217,7 @@ pthread_rwlock_timedrdlock:
>
> 7: movl %edx, %eax
>
> +77:
> addl $8, %esp
> cfi_adjust_cfa_offset(-8)
> popl %ebp
> diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
> index e0fc809..c23f270 100644
> --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
> +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
> @@ -21,7 +21,13 @@
> #include <lowlevelrwlock.h>
> #include <pthread-errnos.h>
> #include <kernel-features.h>
> +#include <hle.h>
>
> +#ifdef PIC
> +#define MO(x) x##@GOTOFF(%edx)
> +#else
> +#define MO(x) x
> +#endif
>
> .text
>
> @@ -48,6 +54,44 @@ pthread_rwlock_timedwrlock:
> movl 28(%esp), %ebp
> movl 32(%esp), %edi
>
> +#ifdef PIC
> + SETUP_PIC_REG(dx)
> + addl $_GLOBAL_OFFSET_TABLE_,%edx
> +#endif
> + cmpb $0,RW_ELISION(%ebp)
> + js not_elided_twrlock
> + jnz try_trans_wrlock
> + /* zero: use default */
> +
> + cmpl $0,MO(__rwlock_rtm_enabled)
> + jz not_elided_twrlock
> +
> +try_trans_wrlock:
> + XBEGIN abort_twrlock
> +
> + /* Lock writer free? */
> + cmpl $0,WRITER(%ebp)
> + jnz 1f
> + cmpl $0,NR_READERS(%ebp)
> + jnz 1f
> +
> + /* Lock is free. Run with transaction */
> + xor %eax,%eax
> +
> + jmp 77f
> +
> + /* Lock is not free. Run */
> +1: XABORT 0xff
> + jmp not_elided_twrlock
> +
> + /* Abort happened. */
> +abort_twrlock:
> +
> + /* Otherwise we just fall back directly to the lock.
> + Here's the place to add more adaptation. */
> +
> +not_elided_twrlock:
> +
> /* Get the lock. */
> movl $1, %edx
> xorl %eax, %eax
> @@ -156,7 +200,7 @@ pthread_rwlock_timedwrlock:
>
> 7: movl %edx, %eax
>
> - addl $8, %esp
> +77: addl $8, %esp
> cfi_adjust_cfa_offset(-8)
> popl %ebp
> cfi_adjust_cfa_offset(-4)
> diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
> index 708e31c..9b384bb 100644
> --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
> +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
> @@ -20,6 +20,7 @@
> #include <lowlevellock.h>
> #include <lowlevelrwlock.h>
> #include <kernel-features.h>
> +#include <hle.h>
>
>
> .text
> @@ -38,6 +39,24 @@ __pthread_rwlock_unlock:
>
> movl 12(%esp), %edi
>
> + /* Is lock free? */
> + cmpl $0,WRITER(%edi)
> + jnz 1f
> + cmpl $0,NR_READERS(%edi)
> + jnz 1f
> +
> + /* Looks free. Assume transaction.
> + If you crash here you unlocked a free lock. */
> + XEND
> + xor %eax,%eax
> +
> + pop %edi
> + cfi_adjust_cfa_offset(-4)
> + pop %ebx
> + cfi_adjust_cfa_offset(-4)
> + ret
> +
> +1:
> /* Get the lock. */
> movl $1, %edx
> xorl %eax, %eax
> diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
> index 6ea17f7..e0c3428 100644
> --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
> +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
> @@ -21,9 +21,16 @@
> #include <lowlevelrwlock.h>
> #include <pthread-errnos.h>
> #include <kernel-features.h>
> +#include <hle.h>
>
> #include <stap-probe.h>
>
> +#ifdef PIC
> +#define MO(x) x@GOTOFF(%edx)
> +#else
> +#define MO(x) x
> +#endif
> +
> .text
>
> .globl __pthread_rwlock_wrlock
> @@ -43,6 +50,49 @@ __pthread_rwlock_wrlock:
>
> LIBC_PROBE (wrlock_entry, 1, %ebx)
>
> +#ifdef PIC
> + SETUP_PIC_REG(dx)
> + addl $_GLOBAL_OFFSET_TABLE_,%edx
> +#endif
> + cmpb $0,RW_ELISION(%ebx)
> + js not_elided_wrlock
> + jnz try_trans_wrlock
> + /* zero: use default */
> +
> + cmpl $0,MO(__rwlock_rtm_enabled)
> + jz not_elided_wrlock
> +
> +try_trans_wrlock:
> + XBEGIN abort_wrlock
> +
> + /* Lock writer free? */
> + /* Ignore readers because we don't need them */
> + cmpl $0,WRITER(%ebx)
> + jnz 1f
> + cmpl $0,NR_READERS(%ebx)
> + jnz 1f
> +
> + /* Lock is free. Run with transaction */
> + xor %eax,%eax
> +
> + pop %ebx
> + cfi_adjust_cfa_offset(-4)
> + pop %esi
> + cfi_adjust_cfa_offset(-4)
> + ret
> +
> + /* Lock is not free. Run */
> +1: XABORT 0xff
> + jmp not_elided_wrlock
> +
> + /* Abort happened. */
> +abort_wrlock:
> +
> + /* Otherwise we just fall back directly to the lock.
> + Here's the place to add more adaptation. */
> +
> +not_elided_wrlock:
> +
> /* Get the lock. */
> movl $1, %edx
> xorl %eax, %eax
> @@ -179,5 +229,7 @@ __pthread_rwlock_wrlock:
> cfi_endproc
> .size __pthread_rwlock_wrlock,.-__pthread_rwlock_wrlock
>
> +#ifndef __pthread_rwlock_wrlock
> strong_alias (__pthread_rwlock_wrlock, pthread_rwlock_wrlock)
> hidden_def (__pthread_rwlock_wrlock)
> +#endif
> diff --git a/nptl/sysdeps/unix/sysv/linux/internaltypes.h b/nptl/sysdeps/unix/sysv/linux/internaltypes.h
> index 699a618..70aaa40 100644
> --- a/nptl/sysdeps/unix/sysv/linux/internaltypes.h
> +++ b/nptl/sysdeps/unix/sysv/linux/internaltypes.h
> @@ -20,7 +20,7 @@
> #define _INTERNALTYPES_H 1
>
> #include <stdint.h>
> -
> +#include <endian.h>
>
> struct pthread_attr
> {
> @@ -86,7 +86,19 @@ struct pthread_condattr
> struct pthread_rwlockattr
> {
> int lockkind;
> - int pshared;
> + /* This used to be an int pshared. We keep the pshared at the
> + same location.
> + PTHREAD_PROCESS_SHARED and PTHREAD_PROCESS_PRIVATE, both fit into one
> + byte. */
> +#if __BYTE_ORDER == __LITTLE_ENDIAN
> + short pshared;
> + char rw_elision;
> + char pad;
> +#else
> + char rw_elision;
> + char pad;
> + short pshared;
> +#endif
> };
>
>
> diff --git a/nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym b/nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym
> index f50b25b..6d9d201 100644
> --- a/nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym
> +++ b/nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym
> @@ -3,6 +3,10 @@
> #include <bits/pthreadtypes.h>
> #include <bits/wordsize.h>
>
> +#ifndef __PTHREAD_RWLOCK_ELISION
> +#define __rw_elision __shared
> +#endif
> +
> --
>
> MUTEX offsetof (pthread_rwlock_t, __data.__lock)
> @@ -14,3 +18,4 @@ WRITERS_QUEUED offsetof (pthread_rwlock_t, __data.__nr_writers_queued)
> FLAGS offsetof (pthread_rwlock_t, __data.__flags)
> WRITER offsetof (pthread_rwlock_t, __data.__writer)
> PSHARED offsetof (pthread_rwlock_t, __data.__shared)
> +RW_ELISION offsetof (pthread_rwlock_t, __data.__rw_elision)
> diff --git a/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h
> index 1852e07..c3325d9 100644
> --- a/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h
> +++ b/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h
> @@ -183,12 +183,13 @@ typedef union
> unsigned int __nr_writers_queued;
> int __writer;
> int __shared;
> - unsigned long int __pad1;
> + long __rw_elision;
> unsigned long int __pad2;
> /* FLAGS must stay at this position in the structure to maintain
> binary compatibility. */
> unsigned int __flags;
> # define __PTHREAD_RWLOCK_INT_FLAGS_SHARED 1
> +# define __PTHREAD_RWLOCK_ELISION 1
> } __data;
> # else
> struct
> @@ -203,8 +204,9 @@ typedef union
> binary compatibility. */
> unsigned char __flags;
> unsigned char __shared;
> - unsigned char __pad1;
> + char __rw_elision;
> unsigned char __pad2;
> +# define __PTHREAD_RWLOCK_ELISION 2
> int __writer;
> } __data;
> # endif
> diff --git a/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c b/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c
> new file mode 100644
> index 0000000..f18295c
> --- /dev/null
> +++ b/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c
> @@ -0,0 +1,52 @@
> +/* pthread_rwlock_tryrdlock: Lock elision version of pthreads rwlock_tryrdlock.
> + Copyright (C) 2013 Free Software Foundation, Inc.
> + This file is part of the GNU C Library.
> +
> + The GNU C Library is free software; you can redistribute it and/or
> + modify it under the terms of the GNU Lesser General Public
> + License as published by the Free Software Foundation; either
> + version 2.1 of the License, or (at your option) any later version.
> +
> + The GNU C Library is distributed in the hope that it will be useful,
> + but WITHOUT ANY WARRANTY; without even the implied warranty of
> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + Lesser General Public License for more details.
> +
> + You should have received a copy of the GNU Lesser General Public
> + License along with the GNU C Library; if not, see
> + <http://www.gnu.org/licenses/>. */
> +#include <pthread.h>
> +#include <pthreadP.h>
> +#include <hle.h>
> +#include <elision-conf.h>
> +#include "init-arch.h"
> +
> +#define __pthread_rwlock_tryrdlock __full_pthread_rwlock_tryrdlock
> +#include <nptl/pthread_rwlock_tryrdlock.c>
> +#undef __pthread_rwlock_tryrdlock
> +
> +int
> +__pthread_rwlock_tryrdlock(pthread_rwlock_t *rwlock)
> +{
> + unsigned status;
> +
> + if (rwlock->__data.__rw_elision > 0
> + || (rwlock->__data.__rw_elision == 0 && __rwlock_rtm_enabled))
> + {
> + if ((status = _xbegin()) == _XBEGIN_STARTED)
> + {
> + if (rwlock->__data.__writer == 0
> + && rwlock->__data.__nr_readers == 0)
> + return 0;
> + /* Lock was busy. Fall back to normal locking.
> + Could also _xend here but xabort with 0xff code
> + is more visible in the profiler. */
> + _xabort (_ABORT_LOCK_BUSY);
> + }
> + /* Aborts come here */
> + }
> +
> + return __full_pthread_rwlock_tryrdlock (rwlock);
> +}
> +
> +strong_alias(__pthread_rwlock_tryrdlock, pthread_rwlock_tryrdlock);
> diff --git a/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_trywrlock.c b/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_trywrlock.c
> new file mode 100644
> index 0000000..0f5cd3c
> --- /dev/null
> +++ b/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_trywrlock.c
> @@ -0,0 +1,51 @@
> +/* pthread_rwlock_trywrlock: Lock elision version of pthreads rwlock trywrlock.
> + Copyright (C) 2013 Free Software Foundation, Inc.
> + This file is part of the GNU C Library.
> +
> + The GNU C Library is free software; you can redistribute it and/or
> + modify it under the terms of the GNU Lesser General Public
> + License as published by the Free Software Foundation; either
> + version 2.1 of the License, or (at your option) any later version.
> +
> + The GNU C Library is distributed in the hope that it will be useful,
> + but WITHOUT ANY WARRANTY; without even the implied warranty of
> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + Lesser General Public License for more details.
> +
> + You should have received a copy of the GNU Lesser General Public
> + License along with the GNU C Library; if not, see
> + <http://www.gnu.org/licenses/>. */
> +#include <pthread.h>
> +#include <pthreadP.h>
> +#include <hle.h>
> +#include <elision-conf.h>
> +#include "init-arch.h"
> +
> +#define __pthread_rwlock_trywrlock __full_pthread_rwlock_trywrlock
> +#include <nptl/pthread_rwlock_trywrlock.c>
> +#undef __pthread_rwlock_trywrlock
> +
> +int
> +__pthread_rwlock_trywrlock(pthread_rwlock_t *rwlock)
> +{
> + unsigned status;
> +
> + if ((rwlock->__data.__rw_elision == 0 && __rwlock_rtm_enabled)
> + || rwlock->__data.__rw_elision > 0)
> + {
> + _xabort (_ABORT_NESTED_TRYLOCK);
> + if ((status = _xbegin()) == _XBEGIN_STARTED)
> + {
> + if (rwlock->__data.__writer == 0
> + && rwlock->__data.__nr_readers == 0)
> + return 0;
> + /* Lock was busy. Fall back to normal locking.
> + Could also _xend here but xabort with 0xff code
> + is more visible in the profiler. */
> + _xabort (_ABORT_LOCK_BUSY);
> + }
> + }
> +
> + return __full_pthread_rwlock_trywrlock (rwlock);
> +}
> +strong_alias(__pthread_rwlock_trywrlock, pthread_rwlock_trywrlock);
> diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S
> index 7681818..3b335bc 100644
> --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S
> +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S
> @@ -22,6 +22,7 @@
> #include <pthread-errnos.h>
> #include <kernel-features.h>
> #include <stap-probe.h>
> +#include <hle.h>
>
> .text
>
> @@ -33,6 +34,51 @@ __pthread_rwlock_rdlock:
>
> LIBC_PROBE (rdlock_entry, 1, %rdi)
>
> + cmpq $0,RW_ELISION(%rdi)
> + js not_elided_rdlock
> + jnz 2f
> + /* zero: use default */
> +
> + cmpl $0,__rwlock_rtm_enabled(%rip)
> + jz not_elided_rdlock
> +
> +2:
> + mov __rwlock_rtm_read_retries(%rip),%esi
> +
> +try_trans_rdlock:
> + XBEGIN abort_rdlock
> +
> + /* Lock reader/writer free? */
> + cmpl $0,WRITER(%rdi)
> + jnz 1f
> + cmpl $0,NR_READERS(%rdi)
> + jnz 1f
> +
> + /* Lock is free. Run with transaction */
> + xor %eax,%eax
> + ret
> +
> + /* Lock is not free. Run */
> +1: XABORT 0xff
> + jmp not_elided_rdlock
> +
> + /* Abort happened. */
> +abort_rdlock:
> + testl $_XABORT_CONFLICT,%eax
> + jz not_elided_rdlock
> +
> + /* For a reader that aborts due a conflict retry speculation
> + a limited number of times. This way when some reader aborts
> + because the reader count is written the other readers will
> + still elide, at the cost of retrying the speculation. */
> +
> + dec %esi
> + jnz try_trans_rdlock
> +
> + /* Otherwise we just fall back directly to the lock.
> + Here's the place to add more adaptation. */
> +
> +not_elided_rdlock:
> xorq %r10, %r10
>
> /* Get the lock. */
> @@ -173,5 +219,7 @@ __pthread_rwlock_rdlock:
> cfi_endproc
> .size __pthread_rwlock_rdlock,.-__pthread_rwlock_rdlock
>
> +#ifndef __pthread_rwlock_rdlock
> strong_alias (__pthread_rwlock_rdlock, pthread_rwlock_rdlock)
> hidden_def (__pthread_rwlock_rdlock)
> +#endif
> diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S
> index 57fe1e9..2bc9037 100644
> --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S
> +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S
> @@ -21,6 +21,7 @@
> #include <lowlevelrwlock.h>
> #include <pthread-errnos.h>
> #include <kernel-features.h>
> +#include <hle.h>
>
> .text
>
> @@ -29,6 +30,53 @@
> .align 16
> pthread_rwlock_timedrdlock:
> cfi_startproc
> +
> + cmpq $0,RW_ELISION(%rdi)
> + js not_elided_timedrdlock
> + jnz 2f
> + /* zero: use default */
> +
> + cmpl $0,__rwlock_rtm_enabled(%rip)
> + jz not_elided_timedrdlock
> +
> +2:
> + mov __rwlock_rtm_read_retries(%rip),%ecx
> +
> +try_trans_timedrdlock:
> + XBEGIN abort_timedrdlock
> +
> + /* Lock reader/writer free? */
> + cmpl $0,WRITER(%rdi)
> + jnz 1f
> + cmpl $0,NR_READERS(%rdi)
> + jnz 1f
> +
> + /* Lock is free. Run with transaction */
> + xor %eax,%eax
> + ret
> +
> + /* Lock is not free. Run */
> +1: XABORT 0xff
> + jmp not_elided_timedrdlock
> +
> + /* Abort happened. */
> +abort_timedrdlock:
> + testl $_XABORT_CONFLICT,%eax
> + jz not_elided_timedrdlock
> +
> + /* For a reader that aborts due a conflict retry speculation
> + a limited number of times. This way when some reader aborts
> + because the reader count is written the other readers will
> + still elide, at the cost of retrying the speculation. */
> +
> + dec %ecx
> + jnz try_trans_timedrdlock
> +
> + /* Otherwise we just fall back directly to the lock.
> + Here's the place to add more adaptation. */
> +
> +not_elided_timedrdlock:
> +
> pushq %r12
> cfi_adjust_cfa_offset(8)
> cfi_rel_offset(%r12, 0)
> diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S
> index 391be17..701c07a 100644
> --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S
> +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S
> @@ -21,6 +21,7 @@
> #include <lowlevelrwlock.h>
> #include <pthread-errnos.h>
> #include <kernel-features.h>
> +#include <hle.h>
>
> .text
>
> @@ -29,6 +30,37 @@
> .align 16
> pthread_rwlock_timedwrlock:
> cfi_startproc
> +
> + cmpq $0,RW_ELISION(%rdi)
> + js not_elided_timedwrlock
> + jnz 2f
> + /* zero: use default */
> +
> + cmpl $0,__rwlock_rtm_enabled(%rip)
> + jz not_elided_timedwrlock
> +
> +2:
> + XBEGIN abort
> +
> + /* Lock free? */
> + cmpl $0,NR_READERS(%rdi)
> + jnz 1f
> + cmpl $0,WRITER(%rdi)
> + jnz 1f
> +
> + /* Lock is free. Run with transaction */
> + xor %rax, %rax
> + ret
> +
> + /* Lock is not free. Run */
> +1: XABORT 0xff
> + jmp not_elided_timedwrlock
> +
> + /* Abort happened. */
> +abort:
> + /* No retries here */
> +
> +not_elided_timedwrlock:
> pushq %r12
> cfi_adjust_cfa_offset(8)
> cfi_rel_offset(%r12, 0)
> diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
> index 86dda05..f1b9516 100644
> --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
> +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
> @@ -20,7 +20,7 @@
> #include <lowlevellock.h>
> #include <lowlevelrwlock.h>
> #include <kernel-features.h>
> -
> +#include <hle.h>
>
> .text
>
> @@ -29,6 +29,21 @@
> .align 16
> __pthread_rwlock_unlock:
> cfi_startproc
> +
> + /* Is lock free? */
> + cmpl $0,WRITER(%rdi)
> + jnz 1f
> + cmpl $0,NR_READERS(%rdi)
> + jnz 1f
> +
> + /* Looks free. Assume transaction.
> + If you crash here you unlocked a free lock. */
> + XEND
> + xor %rax,%rax
> + ret
> +
> +1:
> +
> /* Get the lock. */
> movl $1, %esi
> xorl %eax, %eax
> diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S
> index 734bee3..28d1b4a 100644
> --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S
> +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S
> @@ -22,6 +22,7 @@
> #include <pthread-errnos.h>
> #include <kernel-features.h>
> #include <stap-probe.h>
> +#include <hle.h>
>
> .text
>
> @@ -33,6 +34,34 @@ __pthread_rwlock_wrlock:
>
> LIBC_PROBE (wrlock_entry, 1, %rdi)
>
> + cmpq $0,RW_ELISION(%rdi)
> + js not_elided_wrlock
> + jnz 2f
> + /* zero: use default */
> +
> + cmpl $0,__rwlock_rtm_enabled(%rip)
> + jz not_elided_wrlock
> +
> +2:
> + XBEGIN abort_wrlock
> +
> + /* Lock free? */
> + cmpl $0,WRITER(%rdi)
> + jnz 1f
> + cmpl $0,NR_READERS(%rdi)
> + jnz 1f
> +
> + /* Lock is free. Run with transaction */
> + xor %eax,%eax
> + ret
> +
> + /* Lock is not free. End transaction */
> +1: XABORT 0xff
> + jmp not_elided_wrlock
> +
> + /* Abort happened. */
> +abort_wrlock:
> +not_elided_wrlock:
> xorq %r10, %r10
>
> /* Get the lock. */
> @@ -161,5 +190,7 @@ __pthread_rwlock_wrlock:
> cfi_endproc
> .size __pthread_rwlock_wrlock,.-__pthread_rwlock_wrlock
>
> +#ifndef __pthread_rwlock_wrlock
> strong_alias (__pthread_rwlock_wrlock, pthread_rwlock_wrlock)
> hidden_def (__pthread_rwlock_wrlock)
> +#endif
>